def test_system_create_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2',))
def test_system_get_field_dimensioned_variables(self):
"""Test data is appropriately tagged to identify dimensioned variables."""
path = self.get_temporary_file_path('foo.nc')
time = TemporalVariable(value=[1, 2, 3], dimensions='time')
x = Variable(name='x', value=[10, 20], dimensions='x')
y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y')
data1 = Variable(name='data1',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data2 = Variable(name='data2',
value=np.random.rand(3, 4, 2),
dimensions=['time', 'y', 'x'])
data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time'])
field = Field(time=time,
grid=Grid(x, y),
variables=[data1, data2, data3])
field.write(path)
# Test dimensioned variables are read from a file with appropriate metadata.
rd = RequestDataset(path)
self.assertEqual(rd.variable, ('data1', 'data2'))
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data1', 'data2'))
# Test dimensioned variables are overloaded.
rd = RequestDataset(path, variable='data2')
read_field = rd.get()
actual = get_variable_names(read_field.data_variables)
self.assertEqual(actual, ('data2', ))
def test_load_geometry_subset(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
states = self.get_2d_state_boundaries_sdim()
ca = states[:,states.properties['STATE_NAME'] == 'California']
self.assertTrue(ca.properties['STATE_NAME'] == 'California')
ca.crs.unwrap(ca)
ca = ca.geom.polygon.value[0,0]
for u in [True,False]:
try:
rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo')
field = rd.get()
ca_sub = field.get_intersects(ca,use_spatial_index=u)
self.assertEqual(ca_sub.shape,(1, 3650, 1, 5, 4))
self.assertTrue(ca_sub.variables['foo'].value.mask.any())
self.assertFalse(field.spatial.uid.mask.any())
self.assertFalse(field.spatial.get_mask().any())
ca_sub = field.get_intersects(ca.envelope,use_spatial_index=u)
self.assertEqual(ca_sub.shape,(1, 3650, 1, 5, 4))
self.assertFalse(ca_sub.variables['foo'].value.mask.any())
rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo',time_region={'year':[2007]})
field = rd.get()
ca_sub = field.get_intersects(ca,use_spatial_index=u)
self.assertEqual(ca_sub.shape,(1, 365, 1, 5, 4))
self.assertEqual(set([2007]),set([d.year for d in ca_sub.temporal.value_datetime]))
except ImportError:
with self.assertRaises(ImportError):
import_module('rtree')
def test_shapefile_through_operations(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
ops = OcgOperations(dataset=rd, output_format='shp')
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(list(field.keys()) + [HeaderName.ID_GEOMETRY], list(field2.keys()))
self.assertEqual((51,), field2.data_variables[0].shape)
def test_shapefile_through_operations(self):
path = ShpCabinet().get_shp_path('state_boundaries')
rd = RequestDataset(path)
field = rd.get()
self.assertIsNone(field.spatial.properties)
ops = OcgOperations(dataset=rd, output_format='shp')
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(field.variables.keys(), field2.variables.keys())
self.assertEqual(field.shape, field2.shape)
def test_shapefile_through_operations_subset(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
self.assertIsNone(field.spatial.properties)
ops = OcgOperations(dataset=rd, output_format='shp', geom=path, select_ugid=[15])
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(field.variables.keys(), field2.variables.keys())
self.assertEqual(tuple([1] * 5), field2.shape)
def test_shapefile_through_operations(self):
path = os.path.join(self.path_bin, 'shp', 'state_boundaries',
'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
ops = OcgOperations(dataset=rd, output_format='shp')
ret = ops.execute()
rd2 = RequestDataset(ret)
field2 = rd2.get()
self.assertAsSetEqual(
list(field.keys()) + [HeaderName.ID_GEOMETRY], list(field2.keys()))
self.assertEqual((51, ), field2.data_variables[0].shape)
def test_write_variable_collection_netcdf4_mpi(self):
# TODO: TEST: Test writing a grouped netCDF file in parallel.
self.add_barrier = False
if not env.USE_NETCDF4_MPI:
raise SkipTest('not env.USE_NETCDF4_MPI')
path = self.create_rank_valued_netcdf()
# if vm.rank == 0:
# self.ncdump(path, header_only=False)
rd = RequestDataset(path, driver='netcdf')
rd.metadata['dimensions']['dist_dim']['dist'] = True
field = rd.get()
# self.barrier_print(field['data'].get_value())
if vm.rank == 0:
actual_path = self.get_temporary_file_path('actual_mpi.nc')
else:
actual_path = None
actual_path = vm.bcast(actual_path)
# self.barrier_print('before field.write')
field.write(actual_path)
# self.barrier_print('after field.write')
if vm.rank == 0:
# self.ncdump(actual_path, header_only=False)
self.assertNcEqual(actual_path, path)
def test_load_bounds_datetime_after_slicing(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri)
field = rd.get()
slced = field[:,10:130,:,4:7,100:37]
self.assertEqual(slced.temporal.bounds_datetime.shape,(120,2))
def test(self):
path1 = self.write_field_data('data1')
path2 = self.write_field_data('data2')
path3 = self.write_field_data('basis_var')
time_range = [datetime(2000, 3, 1), datetime(2000, 3, 31)]
rds = [RequestDataset(p, time_range=time_range) for p in [path1, path2]]
mrd = MultiRequestDataset(rds)
basis = RequestDataset(path3, time_range=[datetime(2000, 8, 1), datetime(2000, 8, 31)])
basis_field = basis.get()
calc = [{'func': 'mfpf',
'name': 'output_mfpf',
'kwds': {'reference': ('data1', 'data2'),
'basis': basis_field}}]
ops = OcgOperations(dataset=mrd, calc=calc)
ret = ops.execute()
actual_field = ret.get_element()
actual_variables = get_variable_names(actual_field.data_variables)
self.assertEqual(actual_variables, ('diff_data1_basis_var', 'diff_data2_basis_var'))
sums = [v.get_value().sum() for v in actual_field.data_variables]
for s in sums:
self.assertAlmostEqual(s, 7.8071042497325145)
def run_system_splitting_unstructured(self, genweights):
env.CLOBBER_UNITS_ON_BOUNDS = False
ufile = self.get_temporary_file_path('ugrid.nc')
resolution = 10.
self.fixture_regular_ugrid_file(ufile, resolution)
src_rd = RequestDataset(ufile,
driver=DriverNetcdfUGRID,
grid_abstraction='point')
# src_rd.inspect()
src_grid = src_rd.get().grid
self.assertEqual(src_grid.abstraction, 'point')
dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical())
gs = GridChunker(src_grid,
dst_grid, (3, 3),
check_contains=False,
src_grid_resolution=10.,
paths=self.fixture_paths,
genweights=genweights,
use_spatial_decomp=True)
gs.write_chunks()
actual = gs.create_full_path_from_template('src_template', index=1)
actual = RequestDataset(actual).get()
self.assertIn(GridChunkerConstants.IndexFile.NAME_SRCIDX_GUID, actual)
def test_load(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri)
field = rd.get()
ds = nc.Dataset(uri,'r')
self.assertEqual(field.level,None)
self.assertEqual(field.spatial.crs,WGS84())
tv = field.temporal.value
test_tv = ds.variables['time'][:]
self.assertNumpyAll(tv,test_tv)
self.assertNumpyAll(field.temporal.bounds,ds.variables['time_bnds'][:])
tdt = field.temporal.value_datetime
self.assertEqual(tdt[4],dt(2001,1,5,12))
self.assertNumpyAll(field.temporal.bounds_datetime[1001],np.array([dt(2003,9,29),dt(2003,9,30)]))
rv = field.temporal.value_datetime[100]
rb = field.temporal.bounds_datetime[100]
self.assertTrue(all([rv > rb[0],rv < rb[1]]))
self.assertEqual(field.temporal.extent_datetime,(datetime.datetime(2001,1,1),datetime.datetime(2011,1,1)))
ds.close()
def test_system_with_time_data(self):
"""Test writing data with a time dimension."""
path = self.get_temporary_file_path('what.shp')
t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time')
geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='time')
field = Field(variables=[t, geom],
dimension_map={
'time': {
'variable': 'time'
},
'geom': {
'variable': 'geom'
}
})
field.write(path,
iter_kwargs={'variable': 'time'},
driver=DriverVector)
rd = RequestDataset(uri=path)
field2 = rd.get()
# netcdftime worthlessness
poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'],
['1-01-02 12:00:00', '1-01-03 12:00:00']]
actual = field2['TIME'].get_value().tolist()
res = [p == actual for p in poss]
self.assertTrue(any(res))
def test_get_field_multifile_load(self):
uri = self.test_data.get_uri('narccap_pr_wrfg_ncep')
rd = RequestDataset(uri, 'pr')
field = rd.get()
self.assertEqual(field.temporal.extent_datetime,
(datetime.datetime(1981, 1, 1, 0, 0), datetime.datetime(1991, 1, 1, 0, 0)))
self.assertAlmostEqual(field.temporal.resolution, 0.125)
def test_load_time_range(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri,time_range=[dt(2005,2,15),dt(2007,4,18)])
field = rd.get()
self.assertEqual(field.temporal.value_datetime[0],dt(2005, 2, 15, 12, 0))
self.assertEqual(field.temporal.value_datetime[-1],dt(2007, 4, 18, 12, 0))
self.assertEqual(field.shape,(1,793,1,64,128))
def test_system_mftime(self):
"""Test a multi-file dataset with varying units on the time variables."""
paths = create_mftime_nc_files(self, with_all_cf=True)
rd = RequestDataset(paths)
field = rd.get()
self.assertIsNone(field.temporal._value)
desired = [0., 1., 2., 366., 367., 368.]
actual = field.temporal.get_value().tolist()
self.assertEqual(actual, desired)
ops = OcgOperations(dataset=rd, output_format=constants.OutputFormatName.NETCDF)
ret = ops.execute()
out_rd = RequestDataset(uri=ret)
self.assertEqual(out_rd.get().temporal.get_value().tolist(), desired)
def test_get_intersects(self):
# test with vector geometries
path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp')
rd = RequestDataset(path)
field = rd.get()
polygon = wkb.loads(
'\x01\x06\x00\x00\x00\x03\x00\x00\x00\x01\x03\x00\x00\x00\x01\x00\x00\x00Y\x00\x00\x00\x06]\xcf\xdfo\xd4Q\xc0AV:L\xd7\xe2D@i\x856A\xbf\xd5Q\xc0^`\xe7\x0ch\xe4D@\xd8\xcb\xd4e\x1c\xd6Q\xc0\xf7\x8a\xf1\xab\x15\xe8D@*}\xf3#i\xd5Q\xc0\xda\x02\x0b\xc7\xcf\xedD@P\xbd\xdch\xeb\xd5Q\xc0R\xacZ\xab\x19\xf0D@Hd\x0bIQ\xd5Q\xc0K\x9e\xe3\'\xb1\xf2D@\xd4\xcd\xb4\xb0\x92\xd8Q\xc0^\xbf\x93a\xb8\xf1D@{u\xecSy\xd8Q\xc0o\xc6\x82\x80X\xfdD@\x80t%\xb5;\xd8Q\xc0\x84\x84\x0e\\\xc1\x01E@)\xe1\xbd\xe5\xd5\xdfQ\xc0Jp\xdd6/\x01E@\xf2\xd9\xdb\xaa\x0f\xf3Q\xc0dX\xfc\x0f\x8c\x00E@\x83\xbd\xf9\x8aY\xf3Q\xc0\x1e\xcd\x14\x15M\x02E@[\xbbY\x02\x14\x06R\xc0Z\xe9\xc5dM\x03E@\xc1\xbd\xad\xe5\xb9\x08R\xc0p\x8d\x1a\'a\x03E@\xfbw`\x10| R\xc0z3\xfd&\xf0\x03E@\xe4\x98\x9a\xf8\x8e$R\xc0r.\xe4%\xdb\x03E@\xb2\x07\xf7\xf7=%R\xc0cs\xba\x07\xc4\x02E@\x81\xa6\xef\x9b\xe6&R\xc0\xf8qV\x1f\xeb\x02E@\xa6~rz\x02\'R\xc0*\xf6\x9b\x9d\xe8\x03E@\xb3\xefU\x92`0R\xc0\xa4SJ\x1cU\x04E@\xb41\x00\xf4\x1f1R\xc0tK0\x05G\x00E@\r\x98h\xdbT4R\xc0\x1c\xc0$\xc5\xa3\xffD@\x03\xa2\xcd\xbc@4R\xc0\xc0\xe2)\xf8I\x04E@\x0cKd\xddc@R\xc0`\xfau\xf4\x9b\x04E@\x03\xd4\x96\xa3\xebBR\xc0\x82\x8en\xd1\xa5\x04E@\x1e\xca\xef\xa0\xfd^R\xc0\x02\xc7\xf9!\x12\x06E@P\xda\xb7\xff\xec_R\xc0\x12\xc1\xa68\xea\tE@2\xe2\x9d\xe7sVR\xc0\x1e\xfb\xe8\xd2\x9b@E@\xb2:3\x0f\x84PR\xc0H-0\xd7~_E@\xf8\x1c\xed\xafBAR\xc0&\xc0\xe3N\xc5^E@\x8c\x1e\x1ec\x12;R\xc0\xe4R\xa1\xf4a^E@5s\nW+\x1dR\xc0\x84\x8b\xf9\xba\xe8\\E@^L\x19*\xea\x11R\xc0\xf8\x16RF8\\E@\x9eZ\xcb\xa9\x88\xfbQ\xc0\xa8\xdb\'\xd6\x85ZE@?+\xbd\t\xa9\xf9Q\xc0~K\x9d\xd6IZE@\x0f\x8f\x0bda\xd2Q\xc0\xee\x90\xcb\xd5kYE@\xec\xa8\x91\x81\'\xd0Q\xc0?\x7fM\xd7\xef\\E@64"\x03d\xcfQ\xc0\xd6\x99\x80\xd2,_E@\xa9\xbb\x11\x1d\xed\xcbQ\xc0\xca\x7f \xb3\x8f^E@4r\xfa\x81\x96\xcbQ\xc0\x9c\x17\xed,VgE@\x9d\xee\xf0\xfa\xb7\xc7Q\xc0\xb2\xd4\x9fq\xbdhE@S\xe2r42\xc4Q\xc0\n\x1c\xe1\xef\xf3fE@!\x83y\x95\xa0\xc1Q\xc0O\x14\xf1.\xf3lE@\xb0\xf2^,\xf7\xbaQ\xc0\xfe\x89\x10\x93LqE@\xf6\x0f\xa9\xa7z\xb9Q\xc0FI\x942\x85qE@\xca>\xfb$b\xb6Q\xc0:\xa9\x7f\xda\x84nE@\x99f=\x9d\x16\xb4Q\xc0b6z\xff\xfbnE@\xf8\x1c\xcc*W\xafQ\xc0\xc8\x1fmU\xeeTE@\xcc\xb7\t\xfa\xf6\xa5Q\xc0p\xa3_"\xbaRE@Qf[{\x8a\xa8Q\xc0J\x07l\x06\x94JE@b\x8c\x1fK\n\xb4Q\xc0 \xffz\xa0\xf1EE@6\xa8\xee\xcf0\xb9Q\xc0:P\xe3MZ9E@\xf1H\xcc\xd5z\xbdQ\xc0\x82\x19u\xaaX7E@n\x18\xea\xb6/\xc2Q\xc0\x175gx\x8f$E@U\xadT\xc7\x15\xbbQ\xc0J_B\xaa\x04\x1eE@\xcb\x88\xa5\x86!\xb9Q\xc0z\xf3\xde\xa1\x04"E@\x1f\x04\x08@\xc7\xb4Q\xc0\x8co.NX!E@?\xce\x01\xf8\x92\xb1Q\xc0\x10\xf6\x91r\xd3\x1fE@nD\xd5\x08\xe8\xabQ\xc0\n\xf6\x9b\xf4\x9a\x13E@\x069\x91\xd5\x98\xa7Q\xc0\xc4\xd5\x10\xa1\xed\xfbD@L\x1b\xef\xe6\x94\xa2Q\xc0$\xad\x14j)\xf7D@\xa4\xe1T\xc3i\xa2Q\xc09\x04\xa28#\xe7D@\xd8\xea\xfa\xce\x1a\x9bQ\xc0g\x01\x88\x04/\xdfD@\xde\xc6#\x80\x86\x91Q\xc0\xff\x88\x16w_\xdcD@\x14hp\x07\xd5\x95Q\xc0(\x95L\xb3\x1c\xdbD@\xa1 \xbe\xf7"\x8dQ\xc0\x19(\xa4\x9a5\xdbD@\x9bKt\xce:\x81Q\xc0\xb8\xe3\x9b\xd3\x08\xe4D@\x175%X\x07\x80Q\xc0\x16\xe7\x88\xe3\x9c\xedD@\x8b\x0e\x11\x8cn\x86Q\xc0pZ\xae\xe7G\x00E@\xef\x08`YT\x90Q\xc0\x90\xc7\xcc\xfd\xb1\x07E@P\x02\xa0Q\xa5\x88Q\xc0\xc2\xf2\xd2~G\tE@\xf4\x84\xd0\xeb:\x83Q\xc0\xbd@\xb0\xbe]\x03E@\xb0G/\xf7\xb4}Q\xc0\xcdI<]\xb9\xf3D@\xae\x04l\xe9\xbczQ\xc0@\x99+wB\xe2D@5\xb6ME\x15}Q\xc0,\xc8f\x8f\xf3\xd5D@\xe6[z\x8br\x99Q\xc0x0\x10\xbdh\xceD@\xc8\x01\xfe\xf2\xb4\x9bQ\xc0\xf0\xb6\xcf\xc6\xed\xc8D@*\x82\xc1\xe3\xc6\xa8Q\xc0\x06n9O\x18\xc5D@\x0ei\x7f\x87\x8d\xaaQ\xc0z\x0cy./\xc7D@\xd8\x12$+\xaa\xa7Q\xc0\x9b\x93\x1bU)\xdeD@\x8f\x07\xb59\xb9\xb5Q\xc0\x9c\xcf\x98t7\xd0D@/\xb9#\xa1\x18\xb9Q\xc0\x94\xc3X\n$\xd1D@\xb2\xeeYk\x13\xc0Q\xc0\xa22ko\x93\xc2D@\xd0PQ\x18\x7f\xc7Q\xc0W\xa9\xe8\xaa\x1c\xbfD@0v(\x9f\t\xc9Q\xc0\xfa\x02g\xff\xdf\xd3D@H\x9dJF\xb9\xccQ\xc0\x0c\xc0\x99\x19\xd9\xd6D@\xb1\xfd\r\x8c\xa7\xceQ\xc0\xa4m\x9f\xba\x95\xdaD@\x96/\xfdo\x10\xd1Q\xc0Xm3\x97\xf7\xdfD@\x06]\xcf\xdfo\xd4Q\xc0AV:L\xd7\xe2D@\x01\x03\x00\x00\x00\x01\x00\x00\x00\x0f\x00\x00\x00\xc0\xb6\x07]\xad\xa6Q\xc02\xbc\x8c5\xff\xb6D@\xa9xP\x1aU\xa4Q\xc02\x92"\xe9v\xbbD@\xf2\xb9_\x96a\xa3Q\xc0o\xeeb\xfbl\xb5D@"\xfdj\xd8\xda\xa4Q\xc0\xc0\x90\x1a;\x84\xb4D@\x186*V\xf8\xa0Q\xc0Z"\x89M\x07\xb3D@\xc7j=\xf0\x1c\x9fQ\xc0\xed3hH\xb8\xabD@\x881\xcdxF\xafQ\xc0zJ`\x9a\xc5\xaaD@L7j\xfbB\xb1Q\xc0h>\xfc?*\xa6D@\xe3\xd2!\xca\x02\xb6Q\xc0\xac!n\xe7\x9e\xacD@p\xbc\xa4\xe0\x14\xb2Q\xc0\x7f\x12\xffI\x1f\xadD@\xbb\x0c\x1b\xdbV\xb1Q\xc0\\\xcd\xe5\xf4\x98\xa9D@\x80\xe8\xd2\xfc\x1c\xb0Q\xc0\x14C\x00\xed\xea\xb0D@\xc7\'\xb0 \xb8\xaaQ\xc0W\x81:c;\xbaD@\x86\x9c\x9f\x1e\xc6\xa6Q\xc02\xfc\xe8\xc4\xc1\xbcD@\xc0\xb6\x07]\xad\xa6Q\xc02\xbc\x8c5\xff\xb6D@\x01\x03\x00\x00\x00\x01\x00\x00\x00\r\x00\x00\x00k\x97\xa4\xa3\x07\x82Q\xc0@\xa7\xf3]\xed\xa7D@\xb8\xaa\xa0\xa1j\x80Q\xc0\x98+\xd84\x92\xa9D@T@x%\xb4\x81Q\xc0x\xd7\x92\xb5)\xabD@R8\x8a\xc8\x9b\x85Q\xc0\xa8\xc2\x93 \xff\xa5D@]r\xdd\x055\x82Q\xc0\xfcUH\x92\xc3\xacD@\xf7"J%\'\x83Q\xc0\xb0)@\xca+\xb2D@\xb8\xfb\xdf\x9e\xd2}Q\xc0\xe1A\x12\x00\xbf\xa5D@\x0f\xd7\xa3\xfd\xfa}Q\xc0\x99\x08\xc8\x84;\xa0D@\x85\xbc\xcfF\x99\x86Q\xc0\x10\xdd1\xf0\x7f\x9eD@eg\xec/\xa6\x8dQ\xc0\x99\xdf\xe4\x16\x96\xa2D@\'\xdc\xad\x10A\x8dQ\xc0\x1ag\xa1\xa7\xa4\xa5D@$\xc4\x04\x8aC\x86Q\xc0Uu\xb2l\x89\xa3D@k\x97\xa4\xa3\x07\x82Q\xc0@\xa7\xf3]\xed\xa7D@')
ret = field.get_intersects(polygon)
self.assertTrue(polygon.almost_equals(ret.spatial.geom.polygon.value.compressed()[0]))
def get_large_vector():
uri = '/home/ben.koziol/ocgis_test_data/nc/CanCM4/tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc'
rd = RequestDataset(uri=uri)
field = rd.get()
print field.shape
with open('/tmp/my_coords.txt', 'w') as f:
f.write(str(field.spatial.grid.value[1].flatten().tolist()))
f.write('/n')
f.write(str(field.spatial.grid.value[0].flatten().tolist()))
def test_load_projection_axes(self):
uri = self.test_data.get_uri('cmip3_extraction')
variable = 'Tavg'
rd = RequestDataset(uri,variable)
with self.assertRaises(DimensionNotFound):
rd.get()
rd = RequestDataset(uri,variable,dimension_map={'R':'projection','T':'time','X':'longitude','Y':'latitude'})
field = rd.get()
self.assertEqual(field.shape,(36, 1800, 1, 7, 12))
self.assertEqual(field.temporal.value_datetime[0],datetime.datetime(1950, 1, 16, 0, 0))
self.assertEqual(field.temporal.value_datetime[-1],datetime.datetime(2099, 12, 15, 0, 0))
self.assertEqual(field.level,None)
self.assertNumpyAll(field.realization.value,np.arange(1,37))
ds = nc.Dataset(uri,'r')
to_test = ds.variables['Tavg']
self.assertNumpyAll(to_test[:],field.variables['Tavg'].value.squeeze())
ds.close()
def test_load_climatology_bounds(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,output_format='nc',geom='state_boundaries',
select_ugid=[27],calc=[{'func':'mean','name':'mean'}],
calc_grouping=['month'])
ret = ops.execute()
rd = RequestDataset(uri=ret,variable='mean')
field = rd.get()
self.assertNotEqual(field.temporal.bounds,None)
def test_esmf(self):
rd1 = RequestDataset(**self.get_dataset())
rd2 = deepcopy(rd1)
ops = OcgOperations(dataset=rd1, regrid_destination=rd2, output_format='nc')
ret = ops.execute()
actual_value = RequestDataset(ret).get().data_variables[0].get_value()
desired_value = rd1.get().data_variables[0].get_value()
self.assertNumpyAllClose(actual_value, desired_value)
def test_get_field_t_conform_units_to(self):
"""
Test conforming time units is appropriately passed to field object.
"""
uri = self.test_data.get_uri('cancm4_tas')
target = get_units_object('days since 1949-1-1', calendar='365_day')
rd = RequestDataset(uri=uri, t_conform_units_to=target)
field = rd.get()
self.assertEqual(field.temporal.conform_units_to, target)
def test_esmf(self):
rd1 = RequestDataset(**self.get_dataset())
rd2 = deepcopy(rd1)
ops = OcgOperations(dataset=rd1,
regrid_destination=rd2,
output_format='nc')
ret = ops.execute()
actual_value = RequestDataset(ret).get().data_variables[0].get_value()
desired_value = rd1.get().data_variables[0].get_value()
self.assertNumpyAllClose(actual_value, desired_value)
def test_load_time_region_slicing(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo',
time_region={'month':[1,10],'year':[2011,2013]})
with self.assertRaises(EmptySubsetError):
rd.get()
rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo',
time_region={'month':[1,10],'year':[2005,2007]})
field = rd.get()
sub = field[:,:,:,50,75]
self.assertEqual(sub.shape,(1,124,1,1,1))
self.assertEqual(sub.variables['foo'].value.shape,(1,124,1,1,1))
field = rd.get()
sub = field[:,:,:,50,75:77]
sub2 = field[:,:,:,0,1]
self.assertEqual(sub2.shape,(1, 124, 1, 1, 1))
def test_get_dimension_map_with_spatial_mask(self):
path = self.get_temporary_file_path('foo.nc')
grid = create_gridxy_global()
gmask = grid.get_mask(create=True)
gmask[1, 1] = True
grid.set_mask(gmask)
grid.parent.write(path)
rd = RequestDataset(path)
driver = DriverNetcdfCF(rd)
dmap = driver.get_dimension_map(driver.metadata_source)
self.assertIsNotNone(dmap.get_spatial_mask())
field = rd.get()
self.assertEqual(field.grid.get_mask().sum(), 1)
# Test mask variable is blown away if set to None during a read.
rd = RequestDataset(path)
rd.dimension_map.set_spatial_mask(None)
self.assertIsNone(rd.dimension_map.get_spatial_mask())
# rd.dimension_map.pprint()
field = rd.get()
self.assertIsNone(field.grid.get_mask())
def test_load_datetime_slicing(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'],uri=uri)
field = rd.get()
field.temporal.value_datetime
field.temporal.bounds_datetime
slced = field[:,239,:,:,:]
self.assertEqual(slced.temporal.value_datetime,np.array([dt(2001,8,28,12)]))
self.assertNumpyAll(slced.temporal.bounds_datetime,np.array([dt(2001,8,28),dt(2001,8,29)]))
def test_load_projection_axes_slicing(self):
uri = self.test_data.get_uri('cmip3_extraction')
variable = 'Tavg'
rd = RequestDataset(uri,variable,dimension_map={'R':'projection','T':'time','X':'longitude','Y':'latitude'})
field = rd.get()
sub = field[15,:,:,:,:]
self.assertEqual(sub.shape,(1,1800,1,7,12))
ds = nc.Dataset(uri,'r')
to_test = ds.variables['Tavg']
self.assertNumpyAll(to_test[15,:,:,:],sub.variables[variable].value.squeeze())
ds.close()
def test_create_dimension_map_with_spatial_mask(self):
path = self.get_temporary_file_path('foo.nc')
grid = create_gridxy_global()
gmask = grid.get_mask(create=True)
gmask[1, 1] = True
grid.set_mask(gmask)
grid.parent.write(path)
rd = RequestDataset(path)
driver = DriverNetcdfCF(rd)
dmap = driver.create_dimension_map(driver.metadata_source)
self.assertIsNotNone(dmap.get_spatial_mask())
field = rd.get()
self.assertEqual(field.grid.get_mask().sum(), 1)
# Test mask variable is blown away if set to None during a read.
rd = RequestDataset(path)
rd.dimension_map.set_spatial_mask(None)
self.assertIsNone(rd.dimension_map.get_spatial_mask())
# rd.dimension_map.pprint()
field = rd.get()
self.assertIsNone(field.grid.get_mask())
def test_crs_with_dimension_map(self):
"""Test CRS overloading in the presence of a dimension map."""
field = self.get_field()
path = self.get_temporary_file_path('foo.nc')
field.write(path)
dmap = {DimensionMapKey.X: {DimensionMapKey.VARIABLE: 'col'},
DimensionMapKey.Y: {DimensionMapKey.VARIABLE: 'row'}}
rd = RequestDataset(path, dimension_map=dmap, crs=Tripole())
field = rd.get()
self.assertIsInstance(field.crs, Tripole)
def get_ofield(self):
rd = RequestDataset(**self.get_dataset())
# Create a field composed of two variables.
ofield = rd.get()
new_variable = ofield['foo'].extract()
new_variable.set_name('foo2')
new_variable.get_value()[:] = 30
ofield.add_variable(new_variable, is_data=True)
ofield = ofield.get_field_slice({'level': 0})
return ofield
def get_ofield(self):
rd = RequestDataset(**self.get_dataset())
# Create a field composed of two variables.
ofield = rd.get()
new_variable = ofield['foo'].extract()
new_variable.set_name('foo2')
new_variable.get_value()[:] = 30
ofield.add_variable(new_variable, is_data=True)
ofield = ofield.get_field_slice({'level': 0})
return ofield
def test_system_spatial_averaging_from_file(self):
rd_nc = self.test_data.get_rd('cancm4_tas')
rd_shp = RequestDataset(self.path_state_boundaries)
field_shp = rd_shp.get()
actual = field_shp.dimension_map.get_variable(DMK.GEOM)
self.assertIsNotNone(actual)
actual = field_shp.dimension_map.get_dimension(DMK.GEOM)
self.assertEqual(len(actual), 1)
self.assertEqual(field_shp.crs, WGS84())
try:
index_geom = np.where(
field_shp['STATE_NAME'].get_value() == 'Nebraska')[0][0]
except IndexError:
# Not found on rank.
polygon_field = None
else:
polygon_field = field_shp.get_field_slice({'geom': index_geom})
polygon_field = MPI_COMM.gather(polygon_field)
if MPI_RANK == 0:
for p in polygon_field:
if p is not None:
polygon_field = p
break
polygon_field = MPI_COMM.bcast(polygon_field)
polygon_field.unwrap()
polygon = polygon_field.geom.get_value()[0]
field_nc = rd_nc.get()
sub_field_nc = field_nc.get_field_slice({'time': slice(0, 10)})
self.assertEqual(sub_field_nc['tas']._dimensions,
field_nc['tas']._dimensions)
sub = sub_field_nc.grid.get_intersects(polygon)
# When split across two processes, there are floating point summing differences.
desired = {1: 2734.5195, 2: 2740.4014}
with vm.scoped_by_emptyable('grid intersects', sub):
if not vm.is_null:
abstraction_geometry = sub.get_abstraction_geometry()
sub.parent.add_variable(abstraction_geometry, force=True)
unioned = abstraction_geometry.get_unioned(
spatial_average='tas')
if unioned is not None:
tas = unioned.parent['tas']
self.assertFalse(tas.is_empty)
self.assertAlmostEqual(tas.get_value().sum(),
desired[vm.size],
places=4)
def test_get_esmf_grid_bilinear_regrid_method(self):
"""Test with a regrid method that does not require corners."""
from ocgis.regrid.base import get_esmf_grid
from ESMF import RegridMethod
import ESMF
rd = RequestDataset(**self.get_dataset())
field = rd.get()
self.assertTrue(field.grid.has_bounds)
egrid = get_esmf_grid(field.grid, regrid_method=RegridMethod.BILINEAR)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
for idx in [0, 1]:
self.assertIsNone(corner[idx])
def test_system_renaming_dimensions_on_variables(self):
var1 = Variable(name='var1', value=[1, 2], dimensions='dim')
var2 = Variable(name='var2', value=[3, 4], dimensions='dim')
vc = VariableCollection(variables=[var1, var2])
path = self.get_temporary_file_path('out.nc')
vc.write(path)
rd = RequestDataset(path)
meta = rd.metadata
meta['dimensions']['new_dim'] = {'size': 2}
meta['variables']['var2']['dimensions'] = ('new_dim',)
field = rd.get()
self.assertEqual(field['var2'].dimensions[0].name, 'new_dim')
self.assertEqual(field['var2'].get_value().tolist(), [3, 4])
path2 = self.get_temporary_file_path('out2.nc')
field.write(path2)
rd2 = RequestDataset(path2)
field2 = rd2.get()
self.assertEqual(field2['var2'].dimensions[0].name, 'new_dim')
self.assertEqual(field2['var2'].get_value().tolist(), [3, 4])
def test_write_variable_collection_parallel(self):
if MPI_RANK == 0:
path1 = self.get_temporary_file_path('out1.shp')
path2 = self.get_temporary_file_path('out2.shp')
else:
path1, path2 = [None] * 2
path1 = MPI_COMM.bcast(path1)
path2 = MPI_COMM.bcast(path2)
# Test writing the field to file.
driver = self.get_driver()
field = driver.create_field()
# Only test open file objects on a single processor.
if MPI_SIZE == 1:
fiona_crs = get_fiona_crs(field)
fiona_schema = get_fiona_schema(field.geom.geom_type,
six.next(field.iter())[1])
fobject = fiona.open(path2,
mode='w',
schema=fiona_schema,
crs=fiona_crs,
driver='ESRI Shapefile')
else:
fobject = None
for target in [path1, fobject]:
# Skip the open file object test during a multi-proc test.
if MPI_SIZE > 1 and target is None:
continue
field.write(target, driver=DriverVector)
if isinstance(target, six.string_types):
path = path1
else:
path = path2
fobject.close()
if MPI_RANK == 0:
with fiona.open(path) as source:
self.assertEqual(len(source), 51)
rd = RequestDataset(uri=path)
field2 = rd.get()
for v in list(field.values()):
if isinstance(v, CoordinateReferenceSystem):
self.assertEqual(v, field2.crs)
else:
self.assertNumpyAll(v.get_value(),
field2[v.name].get_value())
def test_get_field_with_projection(self):
uri = self.test_data.get_uri('narccap_wrfg')
rd = RequestDataset(uri, 'pr')
field = rd.get()
self.assertIsInstance(field.spatial.crs, CFLambertConformal)
field.spatial.update_crs(CFWGS84())
self.assertIsInstance(field.spatial.crs, CFWGS84)
self.assertEqual(field.spatial.grid.row, None)
self.assertAlmostEqual(field.spatial.grid.value.mean(), -26.269666952512416)
field.spatial.crs.unwrap(field.spatial)
self.assertAlmostEqual(field.spatial.grid.value.mean(), 153.73033304748759)
self.assertIsNone(field.spatial.geom.polygon)
self.assertAlmostEqual(field.spatial.geom.point.value[0, 100].x, 278.52630062012787)
self.assertAlmostEqual(field.spatial.geom.point.value[0, 100].y, 21.4615681252577)
def test_get_esmf_grid_bilinear_regrid_method(self):
"""Test with a regrid method that does not require corners."""
from ocgis.regrid.base import create_esmf_grid
from ESMF import RegridMethod
import ESMF
rd = RequestDataset(**self.get_dataset())
field = rd.get()
self.assertTrue(field.grid.has_bounds)
egrid = create_esmf_grid(field.grid, regrid_method=RegridMethod.BILINEAR)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
for idx in [0, 1]:
self.assertIsNone(corner[idx])
def test_get_field(self):
ref_test = self.test_data['cancm4_tas']
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(variable=ref_test['variable'], uri=uri)
field = rd.get()
self.assertIsInstance(field.spatial.grid, NcSpatialGridDimension)
# test names are correctly set when creating the field
self.assertEqual(field.temporal.name, 'time')
self.assertEqual(field.temporal.name_value, 'time')
self.assertEqual(field.temporal.name_bounds, 'time_bnds')
row = field.spatial.grid.row
self.assertEqual(row.name, 'lat')
self.assertEqual(row.name_value, 'lat')
self.assertEqual(row.name_bounds, 'lat_bnds')
col = field.spatial.grid.col
self.assertEqual(col.name, 'lon')
self.assertEqual(col.name_value, 'lon')
self.assertEqual(col.name_bounds, 'lon_bnds')
# test attributes are loaded
self.assertEqual(len(field.attrs), 31)
self.assertEqual(len(field.variables['tas'].attrs), 10)
self.assertEqual(len(field.temporal.attrs), 6)
self.assertEqual(len(field.spatial.grid.row.attrs), 5)
self.assertEqual(len(field.spatial.grid.col.attrs), 5)
ds = nc.Dataset(uri, 'r')
self.assertEqual(field.level, None)
self.assertEqual(field.spatial.crs, WGS84())
tv = field.temporal.value
test_tv = ds.variables['time'][:]
self.assertNumpyAll(tv, test_tv)
self.assertNumpyAll(field.temporal.bounds, ds.variables['time_bnds'][:])
tdt = field.temporal.value_datetime
self.assertEqual(tdt[4], dt(2001, 1, 5, 12))
self.assertNumpyAll(field.temporal.bounds_datetime[1001], np.array([dt(2003, 9, 29), dt(2003, 9, 30)]))
rv = field.temporal.value_datetime[100]
rb = field.temporal.bounds_datetime[100]
self.assertTrue(all([rv > rb[0], rv < rb[1]]))
self.assertEqual(field.temporal.extent_datetime, (datetime.datetime(2001, 1, 1), datetime.datetime(2011, 1, 1)))
ds.close()
def test_get_variable_value(self):
path1 = self.get_temporary_file_path('f1.nc')
path2 = self.get_temporary_file_path('f2.nc')
tdim = Dimension('time')
t1 = TemporalVariable(value=[1, 2, 3, 4, 5], dimensions=tdim, dtype=np.int32)
k = {KeywordArgument.DATASET_KWARGS: {KeywordArgument.FORMAT: 'NETCDF4_CLASSIC'}}
t1.parent.write(path1, **k)
t2 = TemporalVariable(value=[6, 7, 8, 9, 10], dimensions=tdim, dtype=np.int32)
t2.parent.write(path2, **k)
rd = RequestDataset([path1, path2])
tin = rd.get().time
sub = tin[2:8]
self.assertEqual(sub.get_value().tolist(), list(range(3, 9)))
def test_init(self):
# Test passing an open dataset object.
rd = self.get_request_dataset_netcdf()
path = rd.uri
with self.nc_scope(path) as ds:
for _ in range(1):
rd2 = RequestDataset(opened=ds, driver=DriverNetcdf)
field = rd2.get()
self.assertIsInstance(field, Field)
# Test unique identifier.
rd = self.get_request_dataset_netcdf(uid=43)
self.assertEqual(rd.uid, 43)
field = rd.get()
self.assertEqual(field.uid, 43)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_system_parallel_write_ndvariable(self):
"""Test a parallel vector GIS write with a n-dimensional variable."""
ompi = OcgDist()
ompi.create_dimension('time', 3)
ompi.create_dimension('extra', 2)
ompi.create_dimension('x', 4)
ompi.create_dimension('y', 7, dist=True)
ompi.update_dimension_bounds()
if MPI_RANK == 0:
path = self.get_temporary_file_path('foo.shp')
t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time')
t.set_extrapolated_bounds('the_time_bounds', 'bounds')
extra = Variable(name='extra', value=[7, 8], dimensions='extra')
x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float)
x.set_extrapolated_bounds('x_bounds', 'bounds')
# This will have the distributed dimension.
y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float)
y.set_extrapolated_bounds('y_bounds', 'bounds')
data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x'])
dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'},
'y': {'variable': 'y', 'bounds': 'y_bounds'},
'time': {'variable': 'time', 'bounds': 'the_time_bounds'}}
vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data')
vc.set_abstraction_geom()
else:
path, vc = [None] * 2
path = MPI_COMM.bcast(path)
vc = variable_collection_scatter(vc, ompi)
with vm.scoped_by_emptyable('write', vc):
if not vm.is_null:
vc.write(path, driver=DriverVector)
MPI_COMM.Barrier()
desired = 168
rd = RequestDataset(path, driver=DriverVector)
sizes = MPI_COMM.gather(rd.get().geom.shape[0])
if MPI_RANK == 0:
self.assertEqual(sum(sizes), desired)
def test_get_dimension_map_2d_spatial_coordinates(self):
grid = create_gridxy_global()
grid.expand()
path = self.get_temporary_file_path('foo.nc')
f = Field(grid=grid)
f.write(path)
rd = RequestDataset(path)
field = rd.get()
sub = field.get_field_slice({'y': 10, 'x': 5})
self.assertEqual(sub.grid.x.shape, (1, 1))
actual = f.dimension_map.get_dimension(DimensionMapKey.Y)
self.assertEqual(actual, ['y'])
actual = f.dimension_map.get_dimension(DimensionMapKey.X)
self.assertEqual(actual, ['x'])
def test_create_dimension_map_2d_spatial_coordinates(self):
grid = create_gridxy_global()
grid.expand()
path = self.get_temporary_file_path('foo.nc')
f = Field(grid=grid)
f.write(path)
rd = RequestDataset(path)
field = rd.get()
sub = field.get_field_slice({'y': 10, 'x': 5})
self.assertEqual(sub.grid.x.shape, (1, 1))
actual = f.dimension_map.get_dimension(DimensionMapKey.Y)
self.assertEqual(actual, ['y'])
actual = f.dimension_map.get_dimension(DimensionMapKey.X)
self.assertEqual(actual, ['x'])
def test(self):
path = self.get_path_to_template_csv()
path_out = self.get_temporary_file_path('foo_out.csv')
rd = RequestDataset(uri=path)
vc = rd.create_raw_field()
for v in list(vc.values()):
self.assertIsNotNone(v.get_value())
field = rd.get()
self.assertIsInstance(field, Field)
field.write(path_out, driver=DriverCSV)
self.assertCSVFilesEqual(path, path_out)
def test_system_add_variable(self):
"""Test adding variables from spatial collections."""
# Create a few separate fields.
variable_names = tuple(['a', 'b', 'c'])
fields = [self.get_field(variable_name=v) for v in variable_names]
# Create spatial collections containing those fields.
scs = []
for field in fields:
sc = SpatialCollection()
sc.add_field(field, None)
scs.append(sc)
# Destination spatial collection to add variables to from source spatial collections.
grow = scs[0]
# Loop over source fields.
for idx in range(1, len(scs)):
# Loop over child fields and spatial containers in the current source spatial collection.
for field, container in scs[idx].iter_fields(yield_container=True):
# TODO: This should be adjusted to allow easier selection with empty fields.
try:
# Case when we have spatial containers.
grow_field = grow.get_element(field_name=field.name,
container_ugid=container)
except KeyError:
# Case without spatial containers.
grow_field = grow.get_element(field.name)
# Add data variables to the grow field.
for dv in field.data_variables:
grow_field.add_variable(dv.extract(), is_data=True)
# Assert all variables are present on the grow field.
actual = grow.get_element()
self.assertEqual(get_variable_names(actual.data_variables),
variable_names)
# Write the spatial collection using a converter.
conv = NcConverter([grow],
outdir=self.current_dir_output,
prefix='out.nc')
conv.write()
# Assert all variables are present.
rd = RequestDataset(conv.path)
actual = rd.get()
self.assertEqual(get_variable_names(actual.data_variables),
variable_names)
def test_get_esmf_grid(self):
import ESMF
rd = RequestDataset(**self.get_dataset())
have_ocgis_bounds = [True, False]
for h in have_ocgis_bounds:
field = rd.get()
ogrid = field.grid
if not h:
ogrid.remove_bounds()
self.assertFalse(ogrid.has_bounds)
from ocgis.regrid.base import get_esmf_grid
egrid = get_esmf_grid(ogrid)
# ocgis is row major with esmf being column major (i.e. in ocgis rows are stored in the zero index)
for idx_esmf, idx_ocgis in zip([0, 1], [1, 0]):
coords = egrid.coords[ESMF.StaggerLoc.CENTER][idx_esmf]
desired = ogrid.get_value_stacked()[idx_ocgis, :]
self.assertNumpyAll(coords, desired)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
if h:
corner_row = corner[1]
corner_row_actual = np.array([[36.5, 36.5, 36.5, 36.5, 36.5],
[37.5, 37.5, 37.5, 37.5, 37.5],
[38.5, 38.5, 38.5, 38.5, 38.5],
[39.5, 39.5, 39.5, 39.5, 39.5],
[40.5, 40.5, 40.5, 40.5, 40.5]],
dtype=ogrid.archetype.dtype)
self.assertNumpyAll(corner_row, corner_row_actual)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
corner_col = corner[0]
corner_col_actual = np.array(
[[-105.5, -104.5, -103.5, -102.5, -101.5],
[-105.5, -104.5, -103.5, -102.5, -101.5],
[-105.5, -104.5, -103.5, -102.5, -101.5],
[-105.5, -104.5, -103.5, -102.5, -101.5],
[-105.5, -104.5, -103.5, -102.5, -101.5]],
dtype=ogrid.archetype.dtype)
self.assertNumpyAll(corner_col, corner_col_actual)
else:
# No corners should be on the ESMF grid.
for idx in [0, 1]:
self.assertIsNone(corner[idx])
def test_get_ocgis_grid_from_esmf_grid(self):
from ocgis.regrid.base import get_esmf_grid
from ocgis.regrid.base import get_ocgis_grid_from_esmf_grid
rd = RequestDataset(**self.get_dataset())
keywords = dict(has_corners=[True, False],
has_mask=[True, False],
crs=[None, CoordinateReferenceSystem(epsg=4326)])
for k in itr_products_keywords(keywords, as_namedtuple=True):
field = rd.get()
grid = field.grid
if k.has_mask:
gmask = grid.get_mask(create=True)
gmask[1, :] = True
grid.set_mask(gmask)
if not k.has_corners:
grid.x.set_bounds(None)
grid.y.set_bounds(None)
self.assertFalse(grid.has_bounds)
egrid = get_esmf_grid(grid)
ogrid = get_ocgis_grid_from_esmf_grid(egrid, crs=k.crs)
if k.has_mask:
actual_mask = ogrid.get_mask()
self.assertEqual(actual_mask.sum(), 4)
self.assertTrue(actual_mask[1, :].all())
self.assertEqual(ogrid.crs, k.crs)
self.assertNumpyAll(grid.get_value_stacked(),
ogrid.get_value_stacked())
if k.has_corners:
desired = grid.x.bounds.get_value()
actual = ogrid.x.bounds.get_value()
self.assertNumpyAll(actual, desired)
desired = grid.y.bounds.get_value()
actual = ogrid.y.bounds.get_value()
self.assertNumpyAll(actual, desired)
else:
self.assertFalse(ogrid.has_bounds)
def test_get_esmf_grid_change_origin_row_and_col(self):
"""Test with different row and column grid origin."""
from ocgis.regrid.base import get_esmf_grid
import ESMF
rd = RequestDataset(**self.get_dataset())
field = rd.get()
field.grid.y.set_value(np.flipud(field.grid.y.get_value()))
field.grid.y.bounds.set_value(
np.fliplr(np.flipud(field.grid.y.bounds.get_value())))
field.grid.x.set_value(np.flipud(field.grid.x.get_value()))
field.grid.x.bounds.set_value(
np.fliplr(np.flipud(field.grid.x.bounds.get_value())))
egrid = get_esmf_grid(field.grid)
for idx_esmf, idx_ocgis in zip([0, 1], [1, 0]):
coords = egrid.coords[ESMF.StaggerLoc.CENTER][idx_esmf]
self.assertNumpyAll(coords,
field.grid.get_value_stacked()[idx_ocgis, ...])
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
corner_row = corner[1]
dtype = field.grid.dtype
corner_row_actual = np.array(
[[40.5, 40.5, 40.5, 40.5, 40.5], [39.5, 39.5, 39.5, 39.5, 39.5],
[38.5, 38.5, 38.5, 38.5, 38.5], [37.5, 37.5, 37.5, 37.5, 37.5],
[36.5, 36.5, 36.5, 36.5, 36.5]],
dtype=dtype)
self.assertNumpyAll(corner_row, corner_row_actual)
corner = egrid.coords[ESMF.StaggerLoc.CORNER]
corner_col = corner[0]
corner_col_actual = np.array(
[[-101.5, -102.5, -103.5, -104.5, -105.5],
[-101.5, -102.5, -103.5, -104.5, -105.5],
[-101.5, -102.5, -103.5, -104.5, -105.5],
[-101.5, -102.5, -103.5, -104.5, -105.5],
[-101.5, -102.5, -103.5, -104.5, -105.5]],
dtype=dtype)
self.assertNumpyAll(corner_col, corner_col_actual)
def test_system_predicate(self):
"""Test creating a request dataset with a predicate."""
path = self.get_temporary_file_path('foo.nc')
field = self.get_field()
to_exclude = Variable(name='exclude')
field.add_variable(to_exclude)
field.write(path)
rd = RequestDataset(uri=path, predicate=lambda x: not x.startswith('exclude'))
self.assertNotIn('exclude', rd.metadata['variables'])
actual = rd.get()
self.assertNotIn('exclude', actual)
# Test predicate affects data variable identification.
path = self.get_temporary_file_path('foo.nc')
rd = RequestDataset(uri=path, predicate=lambda x: x != 'foo')
with self.assertRaises(NoDataVariablesFound):
assert rd.variable
def test_system_cf_data_read(self):
"""Test some basic reading operations."""
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
self.assertIsInstance(field, Field)
self.assertEqual(rd.variable, 'tas')
self.assertEqual(field['tas'].units, 'K')
self.assertEqual(len(field.dimensions), 4)
self.assertIsNotNone(field.crs)
self.assertIsInstance(field.time, TemporalVariable)
# Geometry is not loaded automatically from the grid.
self.assertIsNone(field.geom)
field.set_abstraction_geom()
self.assertIsInstance(field.geom, GeometryVariable)
# Test overloading units.
path = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(uri=path, units='celsius')
field = rd.get()
self.assertEqual(field['tas'].units, 'celsius')
def test_write_parallel(self):
"""Test writing by selective rank."""
if MPI_SIZE != 3 and MPI_SIZE != 1:
raise SkipTest('MPI_SIZE != 1 or 3')
ranks = list(range(MPI_SIZE))
for base_rank in ranks:
for driver in [DriverCSV, DriverVector, DriverNetcdf]:
if MPI_RANK == 0:
path = self.get_temporary_file_path('{}-{}.{}'.format(
driver.key, base_rank, driver.common_extension))
else:
path = None
path = MPI_COMM.bcast(path)
with vm.scoped('field write by rank', [base_rank]):
if not vm.is_null:
geom = GeometryVariable(
value=[Point(1, 2), Point(3, 4)],
name='geom',
dimensions='geom')
data = Variable(name='data',
value=[10, 20],
dimensions='geom')
field = Field(geom=geom)
field.add_variable(data, is_data=True)
self.assertFalse(os.path.isdir(path))
field.write(path, driver=driver)
self.assertFalse(os.path.isdir(path))
rd = RequestDataset(path, driver=driver)
in_field = rd.get()
self.assertEqual(in_field['data'].dimensions[0].size,
2)
MPI_COMM.Barrier()
MPI_COMM.Barrier()
def test_get_distributed_slice(self):
with vm.scoped('grid write', [0]):
if MPI_RANK == 0:
x = Variable('x', list(range(768)), 'x', float)
y = Variable('y', list(range(768)), 'y', float)
grid = Grid(x, y)
field = Field(grid=grid)
path = self.get_temporary_file_path('grid.nc')
field.write(path)
else:
path = None
path = vm.bcast(path)
rd = RequestDataset(path)
grid = rd.get().grid
bounds_global = deepcopy([d.bounds_global for d in grid.dimensions])
for _ in range(10):
_ = grid.get_distributed_slice([slice(73, 157), slice(305, 386)])
bounds_global_grid_after_slice = [
d.bounds_global for d in grid.dimensions
]
self.assertEqual(bounds_global, bounds_global_grid_after_slice)
def test_keyword_time_subset_func(self):
def _func_(value, bounds=None):
indices = []
for ii, v in enumerate(value.flat):
if v.month == 6:
indices.append(ii)
return indices
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd, time_subset_func=_func_, geom='state_boundaries', geom_select_uid=[20])
ret = ops.execute()
ret = ret.get_element()
for v in ret.temporal.value_datetime:
self.assertEqual(v.month, 6)
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd, time_subset_func=_func_, geom='state_boundaries', geom_select_uid=[20],
output_format=constants.OutputFormatName.NETCDF)
ret = ops.execute()
rd_out = RequestDataset(ret)
for v in rd_out.get().temporal.value_datetime:
self.assertEqual(v.month, 6)
def test_get_intersects_one_rank_with_mask(self):
"""Test mask is created if one rank has a spatial mask."""
if MPI_SIZE != 2:
raise SkipTest('MPI_SIZE != 2')
if MPI_RANK == 0:
value = [1, 2]
else:
value = [3, 4]
ompi = OcgDist()
xdim = ompi.create_dimension('x', 4, dist=True)
ydim = ompi.create_dimension('y', 5, dist=False)
ompi.update_dimension_bounds()
x = Variable('x', value=value, dimensions=xdim)
y = Variable('y', value=[1, 2, 3, 4, 5], dimensions=ydim)
grid = Grid(x, y)
wkt_geom = 'Polygon ((0.72993630573248502 5.22484076433120936, 0.70318471337579691 0.67707006369426814, 2.70063694267515952 0.69490445859872629, 2.59363057324840796 2.54076433121019107, 4.52866242038216527 2.51401273885350296, 4.40382165605095466 5.34968152866241908, 0.72993630573248502 5.22484076433120936))'
subset_geom = wkt.loads(wkt_geom)
sub = grid.get_intersects(subset_geom)
path = self.get_temporary_file_path('foo.nc')
field = Field(grid=sub)
field.write(path)
with vm.scoped('mask count', [0]):
if not vm.is_null:
rd = RequestDataset(path)
out_field = rd.get()
target = out_field[out_field.grid._mask_name].get_value()
select = target != 0
self.assertEqual(select.sum(), 4)
def get_sorted_uris_by_time_dimension(uris, variable=None):
"""
Sort a sequence of NetCDF URIs by the maximum time extent in ascending order.
:param uris: The sequence of NetCDF URIs to sort.
:type uris: list[str]
>>> uris = ['/path/to/file2.nc', 'path/to/file1.nc']
:param str variable: The target variable for sorting. If ``None`` is provided, then the variable will be
autodiscovered.
:returns: A sequence of sorted URIs.
:rtype: list[str]
"""
from ocgis import RequestDataset
to_sort = {}
for uri in uris:
rd = RequestDataset(uri=uri, variable=variable)
to_sort[rd.get().temporal.extent_datetime[1]] = rd.uri
sorted_keys = sorted(to_sort)
ret = [to_sort[sk] for sk in sorted_keys]
return ret
from ocgis import RequestDataset, OcgOperations
from ocgis.contrib.library_icclim import IcclimTG90p
########################################################################################################################
# Compute a custom percentile basis using ICCLIM.
# Path to CF climate dataset. This examples uses the same file for indice and percentile basis calculation.
in_file = '/path/to/cf_data.nc'
# Subset the input dataset to return the desired base period for the percentile basis.
variable = 'tas'
years = range(1971, 2001)
time_region = {'year': years}
rd = RequestDataset(uri=in_file, variable=variable)
field = rd.get()
field.get_time_region(time_region)
# Calculate the percentile basis. The data values must be a three-dimensional array.
arr = field.variables[variable].value.squeeze()
dt_arr = field.temporal.value_datetime
percentile = 90
window_width = 5
percentile_dict = IcclimTG90p.get_percentile_dict(arr, dt_arr, percentile,
window_width)
########################################################################################################################
# Calculate indice using custom percentile basis.
calc = [{
'func': 'icclim_TG90p',
'name': 'TG90p',
