def test_get_field_multifile_load(self):
uri = self.test_data.get_uri('narccap_pr_wrfg_ncep')
rd = RequestDataset(uri, 'pr')
field = rd.get()
self.assertEqual(field.temporal.extent_datetime,
(datetime.datetime(1981, 1, 1, 0, 0), datetime.datetime(1991, 1, 1, 0, 0)))
self.assertAlmostEqual(field.temporal.resolution, 0.125)
def run_system_splitting_unstructured(self, genweights):
env.CLOBBER_UNITS_ON_BOUNDS = False
ufile = self.get_temporary_file_path('ugrid.nc')
resolution = 10.
self.fixture_regular_ugrid_file(ufile, resolution)
src_rd = RequestDataset(ufile,
driver=DriverNetcdfUGRID,
grid_abstraction='point')
# src_rd.inspect()
src_grid = src_rd.get().grid
self.assertEqual(src_grid.abstraction, 'point')
dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical())
gs = GridChunker(src_grid,
dst_grid, (3, 3),
check_contains=False,
src_grid_resolution=10.,
paths=self.fixture_paths,
genweights=genweights,
use_spatial_decomp=True)
gs.write_chunks()
actual = gs.create_full_path_from_template('src_template', index=1)
actual = RequestDataset(actual).get()
self.assertIn(GridChunkerConstants.IndexFile.NAME_SRCIDX_GUID, actual)
def __iter__(self):
non_iterables = [AbstractRequestObject, dict, Field]
if env.USE_ESMF:
from ocgis.regrid.base import ESMF
non_iterables.append(ESMF.Field)
if isinstance(self._value, tuple(non_iterables)):
to_itr = [self._value]
else:
to_itr = self._value
for uid, element in enumerate(to_itr, start=1):
if isinstance(element, dict):
element = RequestDataset(**element)
if env.USE_ESMF and isinstance(element, ESMF.Field):
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
element = get_ocgis_field_from_esmf_field(element)
try:
element = element.copy()
except AttributeError:
element = copy(element)
if element.uid is None:
element.uid = uid
# TODO: Remove me once the driver does not accept request datasets at initialization.
# Try to change the driver UID.
try:
element.driver.rd.uid = uid
except AttributeError:
# The field driver does not keep a copy of the request dataset.
if hasattr(element.driver, 'rd'):
raise
yield element
def test_write_variable_collection(self):
if MPI_RANK == 0:
path_in = self.get_temporary_file_path('foo.nc')
path_out = self.get_temporary_file_path('foo_out.nc')
with self.nc_scope(path_in, 'w') as ds:
ds.createDimension('seven', 7)
var = ds.createVariable('var_seven',
float,
dimensions=('seven', ))
var[:] = np.arange(7, dtype=float) + 10
var.foo = 'bar'
else:
path_in, path_out = [None] * 2
path_in = MPI_COMM.bcast(path_in)
path_out = MPI_COMM.bcast(path_out)
rd = RequestDataset(path_in)
rd.metadata['dimensions']['seven']['dist'] = True
driver = DriverNetcdf(rd)
vc = driver.get_variable_collection()
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path_out)
if MPI_RANK == 0:
self.assertNcEqual(path_in, path_out)
def test_write_variable_fill_value_is_maintained(self):
if vm.size != 4:
raise SkipTest('vm.size != 4')
dist = OcgDist()
dim = dist.create_dimension('dim', 8, dist=True)
dist.update_dimension_bounds()
var = Variable(name='var', dimensions=dim, fill_value=2.)
var.v()[0] = 1
var.v()[1] = 2
var.get_mask(create=True, check_value=True)
if vm.rank == 0:
path = self.get_temporary_file_path('foo.nc')
else:
path = None
path = vm.bcast(path)
var.parent.write(path)
# if vm.rank == 0:
# self.ncdump(path, header_only=False)
with vm.scoped('read test', [0]):
if not vm.is_null:
invar = RequestDataset(path).create_field()['var']
self.assertEqual(invar.get_mask().sum(), 4)
self.assertEqual(invar.fill_value, 2.)
def test(self):
path1 = self.write_field_data('data1')
path2 = self.write_field_data('data2')
path3 = self.write_field_data('basis_var')
time_range = [datetime(2000, 3, 1), datetime(2000, 3, 31)]
rds = [RequestDataset(p, time_range=time_range) for p in [path1, path2]]
mrd = MultiRequestDataset(rds)
basis = RequestDataset(path3, time_range=[datetime(2000, 8, 1), datetime(2000, 8, 31)])
basis_field = basis.get()
calc = [{'func': 'mfpf',
'name': 'output_mfpf',
'kwds': {'reference': ('data1', 'data2'),
'basis': basis_field}}]
ops = OcgOperations(dataset=mrd, calc=calc)
ret = ops.execute()
actual_field = ret.get_element()
actual_variables = get_variable_names(actual_field.data_variables)
self.assertEqual(actual_variables, ('diff_data1_basis_var', 'diff_data2_basis_var'))
sums = [v.get_value().sum() for v in actual_field.data_variables]
for s in sums:
self.assertAlmostEqual(s, 7.8071042497325145)
def test_load_geometry_subset(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
states = self.get_2d_state_boundaries_sdim()
ca = states[:,states.properties['STATE_NAME'] == 'California']
self.assertTrue(ca.properties['STATE_NAME'] == 'California')
ca.crs.unwrap(ca)
ca = ca.geom.polygon.value[0,0]
for u in [True,False]:
try:
rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo')
field = rd.get()
ca_sub = field.get_intersects(ca,use_spatial_index=u)
self.assertEqual(ca_sub.shape,(1, 3650, 1, 5, 4))
self.assertTrue(ca_sub.variables['foo'].value.mask.any())
self.assertFalse(field.spatial.uid.mask.any())
self.assertFalse(field.spatial.get_mask().any())
ca_sub = field.get_intersects(ca.envelope,use_spatial_index=u)
self.assertEqual(ca_sub.shape,(1, 3650, 1, 5, 4))
self.assertFalse(ca_sub.variables['foo'].value.mask.any())
rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo',time_region={'year':[2007]})
field = rd.get()
ca_sub = field.get_intersects(ca,use_spatial_index=u)
self.assertEqual(ca_sub.shape,(1, 365, 1, 5, 4))
self.assertEqual(set([2007]),set([d.year for d in ca_sub.temporal.value_datetime]))
except ImportError:
with self.assertRaises(ImportError):
import_module('rtree')
def test_load(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri)
field = rd.get()
ds = nc.Dataset(uri,'r')
self.assertEqual(field.level,None)
self.assertEqual(field.spatial.crs,WGS84())
tv = field.temporal.value
test_tv = ds.variables['time'][:]
self.assertNumpyAll(tv,test_tv)
self.assertNumpyAll(field.temporal.bounds,ds.variables['time_bnds'][:])
tdt = field.temporal.value_datetime
self.assertEqual(tdt[4],dt(2001,1,5,12))
self.assertNumpyAll(field.temporal.bounds_datetime[1001],np.array([dt(2003,9,29),dt(2003,9,30)]))
rv = field.temporal.value_datetime[100]
rb = field.temporal.bounds_datetime[100]
self.assertTrue(all([rv > rb[0],rv < rb[1]]))
self.assertEqual(field.temporal.extent_datetime,(datetime.datetime(2001,1,1),datetime.datetime(2011,1,1)))
ds.close()
def test_system_changing_field_name(self):
path1 = self.get_temporary_file_path('foo1.nc')
path2 = self.get_temporary_file_path('foo2.nc')
vc1 = VariableCollection(name='vc1')
var1 = Variable('var1', value=[1, 2, 3], dimensions='three', parent=vc1)
vc2 = VariableCollection(name='vc2')
vc1.add_child(vc2)
var2 = Variable('var2', value=[4, 5, 6, 7], dimensions='four', parent=vc2)
vc1.write(path1)
rd = RequestDataset(path1)
# rd.inspect()
nvc = rd.create_raw_field()
nvc2 = nvc.children['vc2']
self.assertIsNone(nvc2['var2']._value)
self.assertEqual(nvc2.name, 'vc2')
nvc2.set_name('extraordinary')
self.assertIsNotNone(nvc2['var2'].get_value())
self.assertEqual(nvc2['var2'].get_value().tolist(), [4, 5, 6, 7])
nvc.write(path2)
rd2 = RequestDataset(path2)
# rd2.inspect()
n2vc = rd2.create_raw_field()
self.assertEqual(n2vc.children[nvc2.name].name, nvc2.name)
def assertWeightFilesEquivalent(self, global_weights_filename, merged_weights_filename):
nwf = RequestDataset(merged_weights_filename).get()
gwf = RequestDataset(global_weights_filename).get()
nwf_row = nwf['row'].get_value()
gwf_row = gwf['row'].get_value()
self.assertAsSetEqual(nwf_row, gwf_row)
nwf_col = nwf['col'].get_value()
gwf_col = gwf['col'].get_value()
self.assertAsSetEqual(nwf_col, gwf_col)
nwf_S = nwf['S'].get_value()
gwf_S = gwf['S'].get_value()
self.assertEqual(nwf_S.sum(), gwf_S.sum())
unique_src = np.unique(nwf_row)
diffs = []
for us in unique_src.flat:
nwf_S_idx = np.where(nwf_row == us)[0]
nwf_col_sub = nwf_col[nwf_S_idx]
nwf_S_sub = nwf_S[nwf_S_idx].sum()
gwf_S_idx = np.where(gwf_row == us)[0]
gwf_col_sub = gwf_col[gwf_S_idx]
gwf_S_sub = gwf_S[gwf_S_idx].sum()
self.assertAsSetEqual(nwf_col_sub, gwf_col_sub)
diffs.append(nwf_S_sub - gwf_S_sub)
diffs = np.abs(diffs)
self.assertLess(diffs.max(), 1e-14)
def __iter__(self):
non_iterables = [AbstractRequestObject, dict, Field]
if env.USE_ESMF:
from ocgis.regrid.base import ESMF
non_iterables.append(ESMF.Field)
if isinstance(self._value, tuple(non_iterables)):
to_itr = [self._value]
else:
to_itr = self._value
for uid, element in enumerate(to_itr, start=1):
if isinstance(element, dict):
element = RequestDataset(**element)
if env.USE_ESMF and isinstance(element, ESMF.Field):
from ocgis.regrid.base import get_ocgis_field_from_esmf_field
element = get_ocgis_field_from_esmf_field(element)
try:
element = element.copy()
except AttributeError:
element = copy(element)
if element.uid is None:
element.uid = uid
# TODO: Remove me once the driver does not accept request datasets at initialization.
# Try to change the driver UID.
try:
element.driver.rd.uid = uid
except AttributeError:
# The field driver does not keep a copy of the request dataset.
if hasattr(element.driver, 'rd'):
raise
yield element
def test_insert_weighted(self):
gs = self.fixture_grid_chunker()
dst_master_path = self.get_temporary_file_path('out.nc')
gs.dst_grid.parent.write(dst_master_path)
dst_master = RequestDataset(dst_master_path).get()
desired_sums = {}
for data_variable in dst_master.data_variables:
dv_sum = data_variable.get_value().sum()
desired_sums[data_variable.name] = dv_sum
self.assertNotEqual(dv_sum, 0)
data_variable.get_value()[:] = 0
dst_master.write(dst_master_path, write_mode=MPIWriteMode.FILL)
dst_master = RequestDataset(dst_master_path).get()
for data_variable in dst_master.data_variables:
self.assertEqual(data_variable.get_value().sum(), 0)
gs.write_chunks()
index_path = gs.create_full_path_from_template('index_file')
gs.insert_weighted(index_path, self.current_dir_output, dst_master_path)
actual_sums = {}
dst_master_inserted = RequestDataset(dst_master_path).get()
for data_variable in dst_master_inserted.data_variables:
dv_value = data_variable.get_value()
dv_sum = dv_value.sum()
actual_sums[data_variable.name] = dv_sum
for k, v in list(actual_sums.items()):
self.assertAlmostEqual(v, desired_sums[k])
def test_system_with_distributed_dimensions_from_file_shapefile(self):
"""Test a distributed read from file."""
path = self.path_state_boundaries
# These are the desired values.
with vm.scoped('desired data write', [0]):
if not vm.is_null:
rd_desired = RequestDataset(uri=path, driver=DriverVector)
var_desired = SourcedVariable(name='STATE_NAME',
request_dataset=rd_desired)
value_desired = var_desired.get_value().tolist()
self.assertEqual(len(value_desired), 51)
rd = RequestDataset(uri=path, driver=DriverVector)
fvar = SourcedVariable(name='STATE_NAME', request_dataset=rd)
self.assertEqual(len(rd.driver.dist.get_group()['dimensions']), 1)
self.assertTrue(fvar.dimensions[0].dist)
self.assertIsNotNone(fvar.get_value())
if MPI_SIZE > 1:
self.assertLessEqual(fvar.shape[0], 26)
values = MPI_COMM.gather(fvar.get_value())
if MPI_RANK == 0:
values = hgather(values)
self.assertEqual(values.tolist(), value_desired)
else:
self.assertIsNone(values)
def test_load_bounds_datetime_after_slicing(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri)
field = rd.get()
slced = field[:,10:130,:,4:7,100:37]
self.assertEqual(slced.temporal.bounds_datetime.shape,(120,2))
def test_system_with_time_data(self):
"""Test writing data with a time dimension."""
path = self.get_temporary_file_path('what.shp')
t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time')
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='time')
field = Field(variables=[t, geom],
dimension_map={
'time': {
'variable': 'time'
},
'geom': {
'variable': 'geom'
}
})
field.write(path,
iter_kwargs={'variable': 'time'},
driver=DriverVector)
rd = RequestDataset(uri=path)
field2 = rd.get()
# netcdftime worthlessness
poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'],
['1-01-02 12:00:00', '1-01-03 12:00:00']]
actual = field2['TIME'].get_value().tolist()
res = [p == actual for p in poss]
self.assertTrue(any(res))
def test_write_variable_collection_netcdf4_mpi(self):
# TODO: TEST: Test writing a grouped netCDF file in parallel.
self.add_barrier = False
if not env.USE_NETCDF4_MPI:
raise SkipTest('not env.USE_NETCDF4_MPI')
path = self.create_rank_valued_netcdf()
# if vm.rank == 0:
# self.ncdump(path, header_only=False)
rd = RequestDataset(path, driver='netcdf')
rd.metadata['dimensions']['dist_dim']['dist'] = True
field = rd.get()
# self.barrier_print(field['data'].get_value())
if vm.rank == 0:
actual_path = self.get_temporary_file_path('actual_mpi.nc')
else:
actual_path = None
actual_path = vm.bcast(actual_path)
# self.barrier_print('before field.write')
field.write(actual_path)
# self.barrier_print('after field.write')
if vm.rank == 0:
# self.ncdump(actual_path, header_only=False)
self.assertNcEqual(actual_path, path)
def test_system_create_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2',))
def test_system_rotated_pole_spherical_subsetting(self):
"""Test rotated pole coordinates are left alone during a subset (no mask applied)."""
def _run_mask_test_(target):
for vn in ['rlat', 'rlon']:
self.assertIsNone(target[vn].get_mask())
rd = RequestDataset(metadata=self.fixture_rotated_spherical_metadata)
field = rd.create_field()
_run_mask_test_(field)
for ctr, vn in enumerate(['lat', 'lon', 'rlat', 'rlon']):
var = field[vn]
var.get_value()[:] = np.arange(var.size).reshape(var.shape) + (ctr * 10)
path = self.get_temporary_file_path('foo.nc')
field.write(path)
new_field = RequestDataset(path).create_field()
_run_mask_test_(new_field)
subset_geom = box(*new_field.grid.extent)
subset_field = new_field.grid.get_intersects(subset_geom, optimized_bbox_subset=True).parent
_run_mask_test_(subset_field)
path2 = self.get_temporary_file_path('foo2.nc')
subset_field.write(path2)
in_subset_field = RequestDataset(path2).create_field()
_run_mask_test_(in_subset_field)
def test_load_time_range(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri,time_range=[dt(2005,2,15),dt(2007,4,18)])
field = rd.get()
self.assertEqual(field.temporal.value_datetime[0],dt(2005, 2, 15, 12, 0))
self.assertEqual(field.temporal.value_datetime[-1],dt(2007, 4, 18, 12, 0))
self.assertEqual(field.shape,(1,793,1,64,128))
def test_init_metadata_only(self):
metadata = {'variables': {'foo': {}}}
rd = RequestDataset(metadata=metadata)
self.assertEqual(rd.driver.key, DriverKey.NETCDF_CF)
self.assertIsNone(rd.uri)
self.assertEqual(rd.metadata, metadata)
field = rd.create_field()
self.assertIn('foo', field.keys())
def test_init(self):
path = self.get_temporary_file_path('foo.nc')
with self.nc_scope(path, 'w') as ds:
ds.createDimension('a', 2)
rd = RequestDataset(uri=path, driver='netcdf')
self.assertIsInstance(rd.driver, DriverNetcdf)
vc = rd.get_variable_collection()
self.assertEqual(len(vc), 0)
def test_init(self):
path = self.get_temporary_file_path('foo.nc')
with self.nc_scope(path, 'w') as ds:
ds.createDimension('a', 2)
rd = RequestDataset(uri=path, driver='netcdf')
self.assertIsInstance(rd.driver, DriverNetcdf)
vc = rd.create_raw_field()
self.assertEqual(len(vc), 0)
def test_redistribute_by_src_idx(self):
if vm.size != 4:
raise SkipTest('vm.size != 4')
dist = OcgDist()
dim1 = dist.create_dimension('dim1', 5 * vm.size, dist=True)
dim2 = dist.create_dimension('dim2', 2, dist=False)
dist.update_dimension_bounds()
rank_value = np.arange(5) + (10 * (vm.rank + 1))
var1 = Variable(name='dvar1', value=rank_value, dimensions=dim1)
var2 = Variable(name='dvar2', dimensions=[dim1, dim2])
var1.parent.add_variable(var2)
path = self.get_temporary_file_path('out.nc')
var1.parent.write(path)
desired_idx = np.array([1, 7, 9, 10, 14])
vdesired_value = variable_gather(var1)
if vm.rank == 0:
desired_value = vdesired_value.get_value()[desired_idx]
desired_idx_ranks = {0: slice(1, 2),
1: [2, 4],
2: [0, 4]}
rd = RequestDataset(path)
rd.metadata['dimensions'][dim1.name]['dist'] = True
field = rd.create_field()
indvar = field[var1.name]
field[var2.name].load()
try:
rank_slice = desired_idx_ranks[vm.rank]
except KeyError:
sub = Variable(is_empty=True)
else:
sub = indvar[rank_slice]
self.barrier_print(sub.is_empty)
redistribute_by_src_idx(indvar, dim1.name, sub.dimensions_dict.get(dim1.name))
with vm.scoped_by_emptyable('gather for test', indvar):
if vm.is_null:
self.assertIn(vm.rank_global, [2, 3])
else:
self.assertIn(vm.rank_global, [0, 1])
for v in [indvar, indvar.parent[var2.name]]:
self.assertIsNone(v._value)
self.assertIsNone(v._mask)
self.assertIsNone(v._is_empty)
self.assertFalse(v._has_initialized_value)
self.rank_print(indvar)
actual_value = variable_gather(indvar)
if vm.rank == 0:
actual_value = actual_value.get_value()
self.assertNumpyAll(actual_value, desired_value)
def test_esmf(self):
rd1 = RequestDataset(**self.get_dataset())
rd2 = deepcopy(rd1)
ops = OcgOperations(dataset=rd1, regrid_destination=rd2, output_format='nc')
ret = ops.execute()
actual_value = RequestDataset(ret).get().data_variables[0].get_value()
desired_value = rd1.get().data_variables[0].get_value()
self.assertNumpyAllClose(actual_value, desired_value)
def get_large_vector():
uri = '/home/ben.koziol/ocgis_test_data/nc/CanCM4/tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc'
rd = RequestDataset(uri=uri)
field = rd.get()
print field.shape
with open('/tmp/my_coords.txt', 'w') as f:
f.write(str(field.spatial.grid.value[1].flatten().tolist()))
f.write('/n')
f.write(str(field.spatial.grid.value[0].flatten().tolist()))
def test_get_intersects(self):
# test with vector geometries
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
polygon = wkb.loads(
'\x01\x06\x00\x00\x00\x03\x00\x00\x00\x01\x03\x00\x00\x00\x01\x00\x00\x00Y\x00\x00\x00\x06]\xcf\xdfo\xd4Q\xc0AV:L\xd7\xe2D@i\x856A\xbf\xd5Q\xc0^`\xe7\x0ch\xe4D@\xd8\xcb\xd4e\x1c\xd6Q\xc0\xf7\x8a\xf1\xab\x15\xe8D@*}\xf3#i\xd5Q\xc0\xda\x02\x0b\xc7\xcf\xedD@P\xbd\xdch\xeb\xd5Q\xc0R\xacZ\xab\x19\xf0D@Hd\x0bIQ\xd5Q\xc0K\x9e\xe3\'\xb1\xf2D@\xd4\xcd\xb4\xb0\x92\xd8Q\xc0^\xbf\x93a\xb8\xf1D@{u\xecSy\xd8Q\xc0o\xc6\x82\x80X\xfdD@\x80t%\xb5;\xd8Q\xc0\x84\x84\x0e\\\xc1\x01E@)\xe1\xbd\xe5\xd5\xdfQ\xc0Jp\xdd6/\x01E@\xf2\xd9\xdb\xaa\x0f\xf3Q\xc0dX\xfc\x0f\x8c\x00E@\x83\xbd\xf9\x8aY\xf3Q\xc0\x1e\xcd\x14\x15M\x02E@[\xbbY\x02\x14\x06R\xc0Z\xe9\xc5dM\x03E@\xc1\xbd\xad\xe5\xb9\x08R\xc0p\x8d\x1a\'a\x03E@\xfbw`\x10| R\xc0z3\xfd&\xf0\x03E@\xe4\x98\x9a\xf8\x8e$R\xc0r.\xe4%\xdb\x03E@\xb2\x07\xf7\xf7=%R\xc0cs\xba\x07\xc4\x02E@\x81\xa6\xef\x9b\xe6&R\xc0\xf8qV\x1f\xeb\x02E@\xa6~rz\x02\'R\xc0*\xf6\x9b\x9d\xe8\x03E@\xb3\xefU\x92`0R\xc0\xa4SJ\x1cU\x04E@\xb41\x00\xf4\x1f1R\xc0tK0\x05G\x00E@\r\x98h\xdbT4R\xc0\x1c\xc0$\xc5\xa3\xffD@\x03\xa2\xcd\xbc@4R\xc0\xc0\xe2)\xf8I\x04E@\x0cKd\xddc@R\xc0`\xfau\xf4\x9b\x04E@\x03\xd4\x96\xa3\xebBR\xc0\x82\x8en\xd1\xa5\x04E@\x1e\xca\xef\xa0\xfd^R\xc0\x02\xc7\xf9!\x12\x06E@P\xda\xb7\xff\xec_R\xc0\x12\xc1\xa68\xea\tE@2\xe2\x9d\xe7sVR\xc0\x1e\xfb\xe8\xd2\x9b@E@\xb2:3\x0f\x84PR\xc0H-0\xd7~_E@\xf8\x1c\xed\xafBAR\xc0&\xc0\xe3N\xc5^E@\x8c\x1e\x1ec\x12;R\xc0\xe4R\xa1\xf4a^E@5s\nW+\x1dR\xc0\x84\x8b\xf9\xba\xe8\\E@^L\x19*\xea\x11R\xc0\xf8\x16RF8\\E@\x9eZ\xcb\xa9\x88\xfbQ\xc0\xa8\xdb\'\xd6\x85ZE@?+\xbd\t\xa9\xf9Q\xc0~K\x9d\xd6IZE@\x0f\x8f\x0bda\xd2Q\xc0\xee\x90\xcb\xd5kYE@\xec\xa8\x91\x81\'\xd0Q\xc0?\x7fM\xd7\xef\\E@64"\x03d\xcfQ\xc0\xd6\x99\x80\xd2,_E@\xa9\xbb\x11\x1d\xed\xcbQ\xc0\xca\x7f \xb3\x8f^E@4r\xfa\x81\x96\xcbQ\xc0\x9c\x17\xed,VgE@\x9d\xee\xf0\xfa\xb7\xc7Q\xc0\xb2\xd4\x9fq\xbdhE@S\xe2r42\xc4Q\xc0\n\x1c\xe1\xef\xf3fE@!\x83y\x95\xa0\xc1Q\xc0O\x14\xf1.\xf3lE@\xb0\xf2^,\xf7\xbaQ\xc0\xfe\x89\x10\x93LqE@\xf6\x0f\xa9\xa7z\xb9Q\xc0FI\x942\x85qE@\xca>\xfb$b\xb6Q\xc0:\xa9\x7f\xda\x84nE@\x99f=\x9d\x16\xb4Q\xc0b6z\xff\xfbnE@\xf8\x1c\xcc*W\xafQ\xc0\xc8\x1fmU\xeeTE@\xcc\xb7\t\xfa\xf6\xa5Q\xc0p\xa3_"\xbaRE@Qf[{\x8a\xa8Q\xc0J\x07l\x06\x94JE@b\x8c\x1fK\n\xb4Q\xc0 \xffz\xa0\xf1EE@6\xa8\xee\xcf0\xb9Q\xc0:P\xe3MZ9E@\xf1H\xcc\xd5z\xbdQ\xc0\x82\x19u\xaaX7E@n\x18\xea\xb6/\xc2Q\xc0\x175gx\x8f$E@U\xadT\xc7\x15\xbbQ\xc0J_B\xaa\x04\x1eE@\xcb\x88\xa5\x86!\xb9Q\xc0z\xf3\xde\xa1\x04"E@\x1f\x04\x08@\xc7\xb4Q\xc0\x8co.NX!E@?\xce\x01\xf8\x92\xb1Q\xc0\x10\xf6\x91r\xd3\x1fE@nD\xd5\x08\xe8\xabQ\xc0\n\xf6\x9b\xf4\x9a\x13E@\x069\x91\xd5\x98\xa7Q\xc0\xc4\xd5\x10\xa1\xed\xfbD@L\x1b\xef\xe6\x94\xa2Q\xc0$\xad\x14j)\xf7D@\xa4\xe1T\xc3i\xa2Q\xc09\x04\xa28#\xe7D@\xd8\xea\xfa\xce\x1a\x9bQ\xc0g\x01\x88\x04/\xdfD@\xde\xc6#\x80\x86\x91Q\xc0\xff\x88\x16w_\xdcD@\x14hp\x07\xd5\x95Q\xc0(\x95L\xb3\x1c\xdbD@\xa1 \xbe\xf7"\x8dQ\xc0\x19(\xa4\x9a5\xdbD@\x9bKt\xce:\x81Q\xc0\xb8\xe3\x9b\xd3\x08\xe4D@\x175%X\x07\x80Q\xc0\x16\xe7\x88\xe3\x9c\xedD@\x8b\x0e\x11\x8cn\x86Q\xc0pZ\xae\xe7G\x00E@\xef\x08`YT\x90Q\xc0\x90\xc7\xcc\xfd\xb1\x07E@P\x02\xa0Q\xa5\x88Q\xc0\xc2\xf2\xd2~G\tE@\xf4\x84\xd0\xeb:\x83Q\xc0\xbd@\xb0\xbe]\x03E@\xb0G/\xf7\xb4}Q\xc0\xcdI<]\xb9\xf3D@\xae\x04l\xe9\xbczQ\xc0@\x99+wB\xe2D@5\xb6ME\x15}Q\xc0,\xc8f\x8f\xf3\xd5D@\xe6[z\x8br\x99Q\xc0x0\x10\xbdh\xceD@\xc8\x01\xfe\xf2\xb4\x9bQ\xc0\xf0\xb6\xcf\xc6\xed\xc8D@*\x82\xc1\xe3\xc6\xa8Q\xc0\x06n9O\x18\xc5D@\x0ei\x7f\x87\x8d\xaaQ\xc0z\x0cy./\xc7D@\xd8\x12$+\xaa\xa7Q\xc0\x9b\x93\x1bU)\xdeD@\x8f\x07\xb59\xb9\xb5Q\xc0\x9c\xcf\x98t7\xd0D@/\xb9#\xa1\x18\xb9Q\xc0\x94\xc3X\n$\xd1D@\xb2\xeeYk\x13\xc0Q\xc0\xa22ko\x93\xc2D@\xd0PQ\x18\x7f\xc7Q\xc0W\xa9\xe8\xaa\x1c\xbfD@0v(\x9f\t\xc9Q\xc0\xfa\x02g\xff\xdf\xd3D@H\x9dJF\xb9\xccQ\xc0\x0c\xc0\x99\x19\xd9\xd6D@\xb1\xfd\r\x8c\xa7\xceQ\xc0\xa4m\x9f\xba\x95\xdaD@\x96/\xfdo\x10\xd1Q\xc0Xm3\x97\xf7\xdfD@\x06]\xcf\xdfo\xd4Q\xc0AV:L\xd7\xe2D@\x01\x03\x00\x00\x00\x01\x00\x00\x00\x0f\x00\x00\x00\xc0\xb6\x07]\xad\xa6Q\xc02\xbc\x8c5\xff\xb6D@\xa9xP\x1aU\xa4Q\xc02\x92"\xe9v\xbbD@\xf2\xb9_\x96a\xa3Q\xc0o\xeeb\xfbl\xb5D@"\xfdj\xd8\xda\xa4Q\xc0\xc0\x90\x1a;\x84\xb4D@\x186*V\xf8\xa0Q\xc0Z"\x89M\x07\xb3D@\xc7j=\xf0\x1c\x9fQ\xc0\xed3hH\xb8\xabD@\x881\xcdxF\xafQ\xc0zJ`\x9a\xc5\xaaD@L7j\xfbB\xb1Q\xc0h>\xfc?*\xa6D@\xe3\xd2!\xca\x02\xb6Q\xc0\xac!n\xe7\x9e\xacD@p\xbc\xa4\xe0\x14\xb2Q\xc0\x7f\x12\xffI\x1f\xadD@\xbb\x0c\x1b\xdbV\xb1Q\xc0\\\xcd\xe5\xf4\x98\xa9D@\x80\xe8\xd2\xfc\x1c\xb0Q\xc0\x14C\x00\xed\xea\xb0D@\xc7\'\xb0 \xb8\xaaQ\xc0W\x81:c;\xbaD@\x86\x9c\x9f\x1e\xc6\xa6Q\xc02\xfc\xe8\xc4\xc1\xbcD@\xc0\xb6\x07]\xad\xa6Q\xc02\xbc\x8c5\xff\xb6D@\x01\x03\x00\x00\x00\x01\x00\x00\x00\r\x00\x00\x00k\x97\xa4\xa3\x07\x82Q\xc0@\xa7\xf3]\xed\xa7D@\xb8\xaa\xa0\xa1j\x80Q\xc0\x98+\xd84\x92\xa9D@T@x%\xb4\x81Q\xc0x\xd7\x92\xb5)\xabD@R8\x8a\xc8\x9b\x85Q\xc0\xa8\xc2\x93 \xff\xa5D@]r\xdd\x055\x82Q\xc0\xfcUH\x92\xc3\xacD@\xf7"J%\'\x83Q\xc0\xb0)@\xca+\xb2D@\xb8\xfb\xdf\x9e\xd2}Q\xc0\xe1A\x12\x00\xbf\xa5D@\x0f\xd7\xa3\xfd\xfa}Q\xc0\x99\x08\xc8\x84;\xa0D@\x85\xbc\xcfF\x99\x86Q\xc0\x10\xdd1\xf0\x7f\x9eD@eg\xec/\xa6\x8dQ\xc0\x99\xdf\xe4\x16\x96\xa2D@\'\xdc\xad\x10A\x8dQ\xc0\x1ag\xa1\xa7\xa4\xa5D@$\xc4\x04\x8aC\x86Q\xc0Uu\xb2l\x89\xa3D@k\x97\xa4\xa3\x07\x82Q\xc0@\xa7\xf3]\xed\xa7D@')
ret = field.get_intersects(polygon)
self.assertTrue(polygon.almost_equals(ret.spatial.geom.polygon.value.compressed()[0]))
def test_load_climatology_bounds(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,output_format='nc',geom='state_boundaries',
select_ugid=[27],calc=[{'func':'mean','name':'mean'}],
calc_grouping=['month'])
ret = ops.execute()
rd = RequestDataset(uri=ret,variable='mean')
field = rd.get()
self.assertNotEqual(field.temporal.bounds,None)
def test_create_merged_weight_file_unstructured(self):
self.remove_dir = False
ufile = self.get_temporary_file_path('ugrid.nc')
resolution = 10.
self.fixture_regular_ugrid_file(ufile, resolution, crs=Spherical())
src_grid = RequestDataset(ufile, driver=DriverNetcdfUGRID, grid_abstraction='point').get().grid
self.assertEqual(src_grid.abstraction, 'point')
dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical())
dst_path = self.get_temporary_file_path('dst.nc')
dst_grid.parent.write(dst_path)
gs = GridSplitter(src_grid, dst_grid, (3, 3), check_contains=False, src_grid_resolution=10.,
paths=self.fixture_paths)
gs.write_subsets()
# Load the grid splitter index file ----------------------------------------------------------------------------
index_path = gs.create_full_path_from_template('index_file')
ifile = RequestDataset(uri=index_path).get()
ifile.load()
gidx = ifile[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE].attrs
source_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_SOURCE_VARIABLE]]
sv = source_filename.join_string_value()
destination_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]]
dv = destination_filename.join_string_value()
# Create weight files for each subset --------------------------------------------------------------------------
for ii, sfn in enumerate(sv):
esp = os.path.join(self.current_dir_output, sfn)
edp = os.path.join(self.current_dir_output, dv[ii])
ewp = gs.create_full_path_from_template('wgt_template', index=ii + 1)
cmd = ['ESMF_RegridWeightGen', '-s', esp, '--src_type', 'UGRID', '--src_meshname',
VariableName.UGRID_HOST_VARIABLE, '-d', edp, '--dst_type', 'GRIDSPEC', '-w', ewp, '--method',
'conserve', '-r', '--no_log']
subprocess.check_call(cmd)
# Merge weight files -------------------------------------------------------------------------------------------
mwf = self.get_temporary_file_path('merged_weight_file.nc')
gs.create_merged_weight_file(mwf)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path('global_weights.nc')
cmd = ['ESMF_RegridWeightGen', '-s', ufile, '--src_type', 'UGRID', '-d', dst_path, '--dst_type',
'GRIDSPEC', '-w', global_weights_filename, '--method', 'conserve', '--weight-only', '--no_log',
'--src_meshname', VariableName.UGRID_HOST_VARIABLE]
subprocess.check_call(cmd)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename, mwf)
def test_shapefile_through_operations(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
ops = OcgOperations(dataset=rd, output_format='shp')
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(list(field.keys()) + [HeaderName.ID_GEOMETRY], list(field2.keys()))
self.assertEqual((51,), field2.data_variables[0].shape)
def test_compute_2d_grid(self):
path = self.get_path_to_2d_grid_netcdf()
rd = RequestDataset(path)
ops = ocgis.OcgOperations(dataset=rd, calc=[{'func': 'mean', 'name': 'mean'}], calc_grouping=['month'],
output_format='nc', add_auxiliary_files=False, geom=[33.7, -35.9, 109.1, 9.4])
ret = compute(ops, 3, verbose=False)
field = RequestDataset(ret).get()
self.assertEqual(field['mean'].shape, (4, 17, 28))
def test_get_field_t_conform_units_to(self):
"""
Test conforming time units is appropriately passed to field object.
"""
uri = self.test_data.get_uri('cancm4_tas')
target = get_units_object('days since 1949-1-1', calendar='365_day')
rd = RequestDataset(uri=uri, t_conform_units_to=target)
field = rd.get()
self.assertEqual(field.temporal.conform_units_to, target)
def test(self):
gs = self.get_grid_splitter()
desired_dst_grid_sum = gs.dst_grid.parent['data'].get_value().sum()
desired_dst_grid_sum = MPI_COMM.gather(desired_dst_grid_sum)
if MPI_RANK == 0:
desired_sum = np.sum(desired_dst_grid_sum)
desired = [{'y': slice(0, 180, None), 'x': slice(0, 240, None)},
{'y': slice(0, 180, None), 'x': slice(240, 480, None)},
{'y': slice(0, 180, None), 'x': slice(480, 720, None)},
{'y': slice(180, 360, None), 'x': slice(0, 240, None)},
{'y': slice(180, 360, None), 'x': slice(240, 480, None)},
{'y': slice(180, 360, None), 'x': slice(480, 720, None)}]
actual = list(gs.iter_dst_grid_slices())
self.assertEqual(actual, desired)
gs.write_subsets()
if MPI_RANK == 0:
rank_sums = []
for ctr in range(1, gs.nsplits_dst[0] * gs.nsplits_dst[1] + 1):
src_path = gs.create_full_path_from_template('src_template', index=ctr)
dst_path = gs.create_full_path_from_template('dst_template', index=ctr)
src_field = RequestDataset(src_path).get()
dst_field = RequestDataset(dst_path).get()
src_envelope_global = box(*src_field.grid.extent_global)
dst_envelope_global = box(*dst_field.grid.extent_global)
self.assertTrue(does_contain(src_envelope_global, dst_envelope_global))
actual = get_variable_names(src_field.data_variables)
self.assertIn('data', actual)
actual = get_variable_names(dst_field.data_variables)
self.assertIn('data', actual)
actual_data_sum = dst_field['data'].get_value().sum()
actual_data_sum = MPI_COMM.gather(actual_data_sum)
if MPI_RANK == 0:
actual_data_sum = np.sum(actual_data_sum)
rank_sums.append(actual_data_sum)
if MPI_RANK == 0:
self.assertAlmostEqual(desired_sum, np.sum(rank_sums))
index_path = gs.create_full_path_from_template('index_file')
self.assertTrue(os.path.exists(index_path))
MPI_COMM.Barrier()
index_path = gs.create_full_path_from_template('index_file')
index_field = RequestDataset(index_path).get()
self.assertTrue(len(list(index_field.keys())) > 2)
def test(self):
gs = self.fixture_grid_chunker()
desired_dst_grid_sum = gs.dst_grid.parent['data'].get_value().sum()
desired_dst_grid_sum = MPI_COMM.gather(desired_dst_grid_sum)
if vm.rank == 0:
desired_sum = np.sum(desired_dst_grid_sum)
desired = [{'y': slice(0, 180, None), 'x': slice(0, 240, None)},
{'y': slice(0, 180, None), 'x': slice(240, 480, None)},
{'y': slice(0, 180, None), 'x': slice(480, 720, None)},
{'y': slice(180, 360, None), 'x': slice(0, 240, None)},
{'y': slice(180, 360, None), 'x': slice(240, 480, None)},
{'y': slice(180, 360, None), 'x': slice(480, 720, None)}]
actual = list(gs.iter_dst_grid_slices())
self.assertEqual(actual, desired)
gs.write_chunks()
if vm.rank == 0:
rank_sums = []
for ctr in range(1, gs.nchunks_dst[0] * gs.nchunks_dst[1] + 1):
src_path = gs.create_full_path_from_template('src_template', index=ctr)
dst_path = gs.create_full_path_from_template('dst_template', index=ctr)
src_field = RequestDataset(src_path).get()
dst_field = RequestDataset(dst_path).get()
src_envelope_global = box(*src_field.grid.extent_global)
dst_envelope_global = box(*dst_field.grid.extent_global)
self.assertTrue(does_contain(src_envelope_global, dst_envelope_global))
actual = get_variable_names(src_field.data_variables)
self.assertIn('data', actual)
actual = get_variable_names(dst_field.data_variables)
self.assertIn('data', actual)
actual_data_sum = dst_field['data'].get_value().sum()
actual_data_sum = MPI_COMM.gather(actual_data_sum)
if MPI_RANK == 0:
actual_data_sum = np.sum(actual_data_sum)
rank_sums.append(actual_data_sum)
if vm.rank == 0:
self.assertAlmostEqual(desired_sum, np.sum(rank_sums))
index_path = gs.create_full_path_from_template('index_file')
self.assertTrue(os.path.exists(index_path))
vm.barrier()
index_path = gs.create_full_path_from_template('index_file')
index_field = RequestDataset(index_path).get()
self.assertTrue(len(list(index_field.keys())) > 2)
def test_esmf(self):
rd1 = RequestDataset(**self.get_dataset())
rd2 = deepcopy(rd1)
ops = OcgOperations(dataset=rd1,
regrid_destination=rd2,
output_format='nc')
ret = ops.execute()
actual_value = RequestDataset(ret).get().data_variables[0].get_value()
desired_value = rd1.get().data_variables[0].get_value()
self.assertNumpyAllClose(actual_value, desired_value)
def test_shapefile_through_operations(self):
path = ShpCabinet().get_shp_path('state_boundaries')
rd = RequestDataset(path)
field = rd.get()
self.assertIsNone(field.spatial.properties)
ops = OcgOperations(dataset=rd, output_format='shp')
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(field.variables.keys(), field2.variables.keys())
self.assertEqual(field.shape, field2.shape)
def test_shapefile_through_operations_subset(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
self.assertIsNone(field.spatial.properties)
ops = OcgOperations(dataset=rd, output_format='shp', geom=path, select_ugid=[15])
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(field.variables.keys(), field2.variables.keys())
self.assertEqual(tuple([1] * 5), field2.shape)
def test_load_datetime_slicing(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri)
field = rd.get()
field.temporal.value_datetime
field.temporal.bounds_datetime
slced = field[:,239,:,:,:]
self.assertEqual(slced.temporal.value_datetime,np.array([dt(2001,8,28,12)]))
self.assertNumpyAll(slced.temporal.bounds_datetime,np.array([dt(2001,8,28),dt(2001,8,29)]))
def test_system_through_operations(self):
calc = [{'func': MockMultiParamFunction.key, 'name': 'my_mvp', 'kwds': self.parms_for_test}]
ops = OcgOperations(dataset=self.fields_for_ops_test, calc=calc)
ret = ops.execute()
actual_variable = ret.get_element(variable_name='my_mvp')
self.assertEqual(actual_variable.get_value().tolist(), self.desired_value)
ops = OcgOperations(dataset=self.fields_for_ops_test, calc=calc, output_format='nc')
ret = ops.execute()
actual = RequestDataset(ret).get()['my_mvp']
self.assertEqual(actual.get_value().tolist(), self.desired_value)
def test_crs_with_dimension_map(self):
"""Test CRS overloading in the presence of a dimension map."""
field = self.get_field()
path = self.get_temporary_file_path('foo.nc')
field.write(path)
dmap = {DimensionMapKey.X: {DimensionMapKey.VARIABLE: 'col'},
DimensionMapKey.Y: {DimensionMapKey.VARIABLE: 'row'}}
rd = RequestDataset(path, dimension_map=dmap, crs=Tripole())
field = rd.get()
self.assertIsInstance(field.crs, Tripole)
def test_load_projection_axes_slicing(self):
uri = self.test_data.get_uri('cmip3_extraction')
variable = 'Tavg'
rd = RequestDataset(uri,variable,dimension_map={'R':'projection','T':'time','X':'longitude','Y':'latitude'})
field = rd.get()
sub = field[15,:,:,:,:]
self.assertEqual(sub.shape,(1,1800,1,7,12))
ds = nc.Dataset(uri,'r')
to_test = ds.variables['Tavg']
self.assertNumpyAll(to_test[15,:,:,:],sub.variables[variable].value.squeeze())
ds.close()
def test_open(self):
# Test with a multi-file dataset.
path1 = self.get_temporary_file_path('foo1.nc')
path2 = self.get_temporary_file_path('foo2.nc')
for idx, path in enumerate([path1, path2]):
with self.nc_scope(path, 'w', format='NETCDF4_CLASSIC') as ds:
ds.createDimension('a', None)
b = ds.createVariable('b', np.int32, ('a', ))
b[:] = idx
uri = [path1, path2]
rd = RequestDataset(uri=uri, driver=DriverNetcdf)
field = rd.get_variable_collection()
self.assertEqual(field['b'].get_value().tolist(), [0, 1])
def test_open(self):
# Test with a multi-file dataset.
path1 = self.get_temporary_file_path('foo1.nc')
path2 = self.get_temporary_file_path('foo2.nc')
for idx, path in enumerate([path1, path2]):
with self.nc_scope(path, 'w', format='NETCDF4_CLASSIC') as ds:
ds.createDimension('a', None)
b = ds.createVariable('b', np.int32, ('a',))
b[:] = idx
uri = [path1, path2]
rd = RequestDataset(uri=uri, driver=DriverNetcdf)
field = rd.create_raw_field()
self.assertEqual(field['b'].get_value().tolist(), [0, 1])
def get_ofield(self):
rd = RequestDataset(**self.get_dataset())
# Create a field composed of two variables.
ofield = rd.get()
new_variable = ofield['foo'].extract()
new_variable.set_name('foo2')
new_variable.get_value()[:] = 30
ofield.add_variable(new_variable, is_data=True)
ofield = ofield.get_field_slice({'level': 0})
return ofield
def get_ofield(self):
rd = RequestDataset(**self.get_dataset())
# Create a field composed of two variables.
ofield = rd.get()
new_variable = ofield['foo'].extract()
new_variable.set_name('foo2')
new_variable.get_value()[:] = 30
ofield.add_variable(new_variable, is_data=True)
ofield = ofield.get_field_slice({'level': 0})
return ofield
def test_system_spatial_averaging_from_file(self):
rd_nc = self.test_data.get_rd('cancm4_tas')
rd_shp = RequestDataset(self.path_state_boundaries)
field_shp = rd_shp.get()
actual = field_shp.dimension_map.get_variable(DMK.GEOM)
self.assertIsNotNone(actual)
actual = field_shp.dimension_map.get_dimension(DMK.GEOM)
self.assertEqual(len(actual), 1)
self.assertEqual(field_shp.crs, WGS84())
try:
index_geom = np.where(
field_shp['STATE_NAME'].get_value() == 'Nebraska')[0][0]
except IndexError:
# Not found on rank.
polygon_field = None
else:
polygon_field = field_shp.get_field_slice({'geom': index_geom})
polygon_field = MPI_COMM.gather(polygon_field)
if MPI_RANK == 0:
for p in polygon_field:
if p is not None:
polygon_field = p
break
polygon_field = MPI_COMM.bcast(polygon_field)
polygon_field.unwrap()
polygon = polygon_field.geom.get_value()[0]
field_nc = rd_nc.get()
sub_field_nc = field_nc.get_field_slice({'time': slice(0, 10)})
self.assertEqual(sub_field_nc['tas']._dimensions,
field_nc['tas']._dimensions)
sub = sub_field_nc.grid.get_intersects(polygon)
# When split across two processes, there are floating point summing differences.
desired = {1: 2734.5195, 2: 2740.4014}
with vm.scoped_by_emptyable('grid intersects', sub):
if not vm.is_null:
abstraction_geometry = sub.get_abstraction_geometry()
sub.parent.add_variable(abstraction_geometry, force=True)
unioned = abstraction_geometry.get_unioned(
spatial_average='tas')
if unioned is not None:
tas = unioned.parent['tas']
self.assertFalse(tas.is_empty)
self.assertAlmostEqual(tas.get_value().sum(),
desired[vm.size],
places=4)
def test_create_merged_weight_file(self):
path_src = self.get_temporary_file_path('src.nc')
path_dst = self.get_temporary_file_path('dst.nc')
src_grid = create_gridxy_global(resolution=30.0, wrapped=False, crs=Spherical())
dst_grid = create_gridxy_global(resolution=35.0, wrapped=False, crs=Spherical())
src_grid.write(path_src)
dst_grid.write(path_dst)
# Split source and destination grids ---------------------------------------------------------------------------
gs = GridSplitter(src_grid, dst_grid, (2, 2), check_contains=False, allow_masked=True, paths=self.fixture_paths)
gs.write_subsets()
# Load the grid splitter index file ----------------------------------------------------------------------------
index_filename = gs.create_full_path_from_template('index_file')
ifile = RequestDataset(uri=index_filename).get()
ifile.load()
gidx = ifile[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE].attrs
source_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_SOURCE_VARIABLE]]
sv = source_filename.join_string_value()
destination_filename = ifile[gidx[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]]
dv = destination_filename.join_string_value()
# Create weight files for each subset --------------------------------------------------------------------------
for ii, sfn in enumerate(sv):
esp = os.path.join(self.current_dir_output, sfn)
edp = os.path.join(self.current_dir_output, dv[ii])
ewp = gs.create_full_path_from_template('wgt_template', index=ii + 1)
cmd = ['ESMF_RegridWeightGen', '-s', esp, '--src_type', 'GRIDSPEC', '-d', edp, '--dst_type',
'GRIDSPEC', '-w', ewp, '--method', 'conserve', '-r', '--no_log']
subprocess.check_call(cmd)
# Merge weight files -------------------------------------------------------------------------------------------
merged_weight_filename = self.get_temporary_file_path('merged_weights.nc')
gs.create_merged_weight_file(merged_weight_filename)
# Generate a global weight file using ESMF ---------------------------------------------------------------------
global_weights_filename = self.get_temporary_file_path('global_weights.nc')
cmd = ['ESMF_RegridWeightGen', '-s', path_src, '--src_type', 'GRIDSPEC', '-d', path_dst, '--dst_type',
'GRIDSPEC', '-w', global_weights_filename, '--method', 'conserve', '--weight-only', '--no_log']
subprocess.check_call(cmd)
# Test merged and global weight files are equivalent -----------------------------------------------------------
self.assertWeightFilesEquivalent(global_weights_filename, merged_weight_filename)
def test_write_variable_collection_parallel(self):
if MPI_RANK == 0:
path1 = self.get_temporary_file_path('out1.shp')
path2 = self.get_temporary_file_path('out2.shp')
else:
path1, path2 = [None] * 2
path1 = MPI_COMM.bcast(path1)
path2 = MPI_COMM.bcast(path2)
# Test writing the field to file.
driver = self.get_driver()
field = driver.create_field()
# Only test open file objects on a single processor.
if MPI_SIZE == 1:
fiona_crs = get_fiona_crs(field)
fiona_schema = get_fiona_schema(field.geom.geom_type,
six.next(field.iter())[1])
fobject = fiona.open(path2,
mode='w',
schema=fiona_schema,
crs=fiona_crs,
driver='ESRI Shapefile')
else:
fobject = None
for target in [path1, fobject]:
# Skip the open file object test during a multi-proc test.
if MPI_SIZE > 1 and target is None:
continue
field.write(target, driver=DriverVector)
if isinstance(target, six.string_types):
path = path1
else:
path = path2
fobject.close()
if MPI_RANK == 0:
with fiona.open(path) as source:
self.assertEqual(len(source), 51)
rd = RequestDataset(uri=path)
field2 = rd.get()
for v in list(field.values()):
if isinstance(v, CoordinateReferenceSystem):
self.assertEqual(v, field2.crs)
else:
self.assertNumpyAll(v.get_value(),
field2[v.name].get_value())
def test_write_variable_collection(self):
# Attempt to write without a geometry variable.
v = Variable('a', value=[1, 2], dimensions='bb')
field = Field(variables=v)
path = self.get_temporary_file_path('out.shp')
with self.assertRaises(ValueError):
field.write(path, driver=DriverVector)
# Test writing a field with two-dimensional geometry storage.
value = [Point(1, 2), Point(3, 4), Point(5, 6), Point(6, 7), Point(8, 9), Point(10, 11)]
gvar = GeometryVariable(value=value, name='points', dimensions='ngeoms')
gvar.reshape([Dimension('lat', 2), Dimension('lon', 3)])
var1 = Variable(name='dummy', value=[6, 7, 8], dimensions=['a'])
var2 = Variable(name='some_lats', value=[41, 41], dimensions=['lat'])
var3 = Variable(name='some_lons', value=[0, 90, 280], dimensions=['lon'])
var4 = Variable(name='data', value=np.random.rand(4, 3, 2), dimensions=['time', 'lon', 'lat'])
field = Field(variables=[var1, var2, var3, var4], geom=gvar, is_data=['data'])
path = self.get_temporary_file_path('2d.shp')
field.write(path, iter_kwargs={'followers': ['some_lats', 'some_lons']}, driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertTrue(len(read) > 2)
self.assertEqual(list(read.keys()),
['data', 'some_lats', 'some_lons', constants.DimensionName.GEOMETRY_DIMENSION])
# Test writing a subset of the variables.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1])
field.write(path, variable_names=['keep'], driver=DriverVector)
read = RequestDataset(uri=path).get()
self.assertNotIn('remove', read)
# Test using append.
path = self.get_temporary_file_path('limited.shp')
value = [Point(1, 2), Point(3, 4), Point(5, 6)]
gvar = GeometryVariable(value=value, name='points', dimensions='points')
var1 = Variable('keep', value=[1, 2, 3], dimensions='points')
var2 = Variable('remove', value=[4, 5, 6], dimensions='points')
field = Field(variables=[var1, var2], geom=gvar, is_data=[var1, var2])
for idx in range(3):
sub = field[{'points': idx}]
if idx == 0:
write_mode = MPIWriteMode.WRITE
else:
write_mode = MPIWriteMode.APPEND
sub.write(path, write_mode=write_mode, driver=DriverVector)
self.assertOGRFileLength(path, idx + 1)
def test_get_esmf_grid_bilinear_regrid_method(self):
"""Test with a regrid method that does not require corners."""
from ocgis.regrid.base import create_esmf_grid
from ESMF import RegridMethod
import ESMF
rd = RequestDataset(**self.get_dataset())
field = rd.get()
self.assertTrue(field.grid.has_bounds)
egrid = create_esmf_grid(field.grid, regrid_method=RegridMethod.BILINEAR)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
for idx in [0, 1]:
self.assertIsNone(corner[idx])
def test_get_esmf_grid_bilinear_regrid_method(self):
"""Test with a regrid method that does not require corners."""
from ocgis.regrid.base import get_esmf_grid
from ESMF import RegridMethod
import ESMF
rd = RequestDataset(**self.get_dataset())
field = rd.get()
self.assertTrue(field.grid.has_bounds)
egrid = get_esmf_grid(field.grid, regrid_method=RegridMethod.BILINEAR)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
for idx in [0, 1]:
self.assertIsNone(corner[idx])
def test_get_field_with_projection(self):
uri = self.test_data.get_uri('narccap_wrfg')
rd = RequestDataset(uri, 'pr')
field = rd.get()
self.assertIsInstance(field.spatial.crs, CFLambertConformal)
field.spatial.update_crs(CFWGS84())
self.assertIsInstance(field.spatial.crs, CFWGS84)
self.assertEqual(field.spatial.grid.row, None)
self.assertAlmostEqual(field.spatial.grid.value.mean(), -26.269666952512416)
field.spatial.crs.unwrap(field.spatial)
self.assertAlmostEqual(field.spatial.grid.value.mean(), 153.73033304748759)
self.assertIsNone(field.spatial.geom.polygon)
self.assertAlmostEqual(field.spatial.geom.point.value[0, 100].x, 278.52630062012787)
self.assertAlmostEqual(field.spatial.geom.point.value[0, 100].y, 21.4615681252577)
def test_chunked_rwg_spatial_subset(self):
env.CLOBBER_UNITS_ON_BOUNDS = False
src_grid = create_gridxy_global(crs=Spherical())
src_field = create_exact_field(src_grid, 'foo')
xvar = Variable(name='x', value=[-90., -80.], dimensions='xdim')
yvar = Variable(name='y', value=[40., 50.], dimensions='ydim')
dst_grid = Grid(x=xvar, y=yvar, crs=Spherical())
if ocgis.vm.rank == 0:
source = self.get_temporary_file_path('source.nc')
else:
source = None
source = ocgis.vm.bcast(source)
src_field.write(source)
if ocgis.vm.rank == 0:
destination = self.get_temporary_file_path('destination.nc')
else:
destination = None
destination = ocgis.vm.bcast(destination)
dst_grid.parent.write(destination)
wd = os.path.join(self.current_dir_output, 'chunks')
weight = os.path.join(self.current_dir_output, 'weights.nc')
spatial_subset = os.path.join(self.current_dir_output,
'spatial_subset.nc')
runner = CliRunner()
cli_args = [
'chunked-rwg', '--source', source, '--destination', destination,
'--wd', wd, '--spatial_subset', '--spatial_subset_path',
spatial_subset, '--weight', weight, '--esmf_regrid_method',
'BILINEAR', '--persist'
]
result = runner.invoke(ocli, args=cli_args, catch_exceptions=False)
self.assertEqual(result.exit_code, 0)
actual = RequestDataset(uri=spatial_subset).create_field()
actual_ymean = actual.grid.get_value_stacked()[0].mean()
actual_xmean = actual.grid.get_value_stacked()[1].mean()
self.assertEqual(actual_ymean, 45.)
self.assertEqual(actual_xmean, -85.)
self.assertEqual(actual.grid.shape, (14, 14))
self.assertTrue(os.path.exists(weight))
actual = RequestDataset(weight, driver='netcdf').create_field()
self.assertIn('history', actual.attrs)
def test_init(self):
# Test passing an open dataset object.
rd = self.get_request_dataset_netcdf()
path = rd.uri
with self.nc_scope(path) as ds:
for _ in range(1):
rd2 = RequestDataset(opened=ds, driver=DriverNetcdf)
field = rd2.get()
self.assertIsInstance(field, Field)
# Test unique identifier.
rd = self.get_request_dataset_netcdf(uid=43)
self.assertEqual(rd.uid, 43)
field = rd.get()
self.assertEqual(field.uid, 43)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def run_op(resource, calc, options):
"""Create an OCGIS operation, launch it and return the results."""
from os.path import abspath, curdir
from ocgis import OcgOperations, RequestDataset, env
import uuid
LOGGER.info('Start ocgis module call function')
# Prepare the environment
env.OVERWRITE = True
dir_output = abspath(curdir)
prefix = str(uuid.uuid1())
env.PREFIX = prefix
rd = [
RequestDataset(val, variable=key if key != 'resource' else None)
for key, val in resource.items()
]
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=options['calc_grouping'],
dir_output=dir_output,
prefix=prefix,
add_auxiliary_files=False,
output_format='nc')
return ops.execute()
def test(self):
path = self.get_path_to_template_csv()
path_out = self.get_temporary_file_path('foo_out.csv')
rd = RequestDataset(uri=path)
vc = rd.create_raw_field()
for v in list(vc.values()):
self.assertIsNotNone(v.get_value())
field = rd.get()
self.assertIsInstance(field, Field)
field.write(path_out, driver=DriverCSV)
self.assertCSVFilesEqual(path, path_out)
def test_mfdataset_to_nc(self):
rd = self.test_data.get_rd('maurer_2010_pr')
ops = OcgOperations(dataset=rd, output_format='nc', calc=[{'func': 'mean', 'name': 'my_mean'}],
calc_grouping=['year'], geom='state_boundaries', select_ugid=[23])
ret = ops.execute()
field = RequestDataset(ret, 'my_mean').get()
self.assertNumpyAll(field.temporal.get_value(), np.array([18444., 18809.]))
