def test_system_create_field_dimensioned_variables(self): """Test data is appropriately tagged to identify dimensioned variables.""" path = self.get_temporary_file_path('foo.nc') time = TemporalVariable(value=[1, 2, 3], dimensions='time') x = Variable(name='x', value=[10, 20], dimensions='x') y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y') data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time']) field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3]) field.write(path) # Test dimensioned variables are read from a file with appropriate metadata. rd = RequestDataset(path) self.assertEqual(rd.variable, ('data1', 'data2')) read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data1', 'data2')) # Test dimensioned variables are overloaded. rd = RequestDataset(path, variable='data2') read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data2',))
def test_system_get_field_dimensioned_variables(self): """Test data is appropriately tagged to identify dimensioned variables.""" path = self.get_temporary_file_path('foo.nc') time = TemporalVariable(value=[1, 2, 3], dimensions='time') x = Variable(name='x', value=[10, 20], dimensions='x') y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y') data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time']) field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3]) field.write(path) # Test dimensioned variables are read from a file with appropriate metadata. rd = RequestDataset(path) self.assertEqual(rd.variable, ('data1', 'data2')) read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data1', 'data2')) # Test dimensioned variables are overloaded. rd = RequestDataset(path, variable='data2') read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data2', ))
def test_load_geometry_subset(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') states = self.get_2d_state_boundaries_sdim() ca = states[:,states.properties['STATE_NAME'] == 'California'] self.assertTrue(ca.properties['STATE_NAME'] == 'California') ca.crs.unwrap(ca) ca = ca.geom.polygon.value[0,0] for u in [True,False]: try: rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo') field = rd.get() ca_sub = field.get_intersects(ca,use_spatial_index=u) self.assertEqual(ca_sub.shape,(1, 3650, 1, 5, 4)) self.assertTrue(ca_sub.variables['foo'].value.mask.any()) self.assertFalse(field.spatial.uid.mask.any()) self.assertFalse(field.spatial.get_mask().any()) ca_sub = field.get_intersects(ca.envelope,use_spatial_index=u) self.assertEqual(ca_sub.shape,(1, 3650, 1, 5, 4)) self.assertFalse(ca_sub.variables['foo'].value.mask.any()) rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo',time_region={'year':[2007]}) field = rd.get() ca_sub = field.get_intersects(ca,use_spatial_index=u) self.assertEqual(ca_sub.shape,(1, 365, 1, 5, 4)) self.assertEqual(set([2007]),set([d.year for d in ca_sub.temporal.value_datetime])) except ImportError: with self.assertRaises(ImportError): import_module('rtree')
def test_shapefile_through_operations(self): path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp') rd = RequestDataset(path) field = rd.get() ops = OcgOperations(dataset=rd, output_format='shp') ret = ops.execute() rd2 = RequestDataset(ret) field2 = rd2.get() self.assertAsSetEqual(list(field.keys()) + [HeaderName.ID_GEOMETRY], list(field2.keys())) self.assertEqual((51,), field2.data_variables[0].shape)
def test_shapefile_through_operations(self): path = ShpCabinet().get_shp_path('state_boundaries') rd = RequestDataset(path) field = rd.get() self.assertIsNone(field.spatial.properties) ops = OcgOperations(dataset=rd, output_format='shp') ret = ops.execute() rd2 = RequestDataset(ret) field2 = rd2.get() self.assertAsSetEqual(field.variables.keys(), field2.variables.keys()) self.assertEqual(field.shape, field2.shape)
def test_shapefile_through_operations_subset(self): path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp') rd = RequestDataset(path) field = rd.get() self.assertIsNone(field.spatial.properties) ops = OcgOperations(dataset=rd, output_format='shp', geom=path, select_ugid=[15]) ret = ops.execute() rd2 = RequestDataset(ret) field2 = rd2.get() self.assertAsSetEqual(field.variables.keys(), field2.variables.keys()) self.assertEqual(tuple([1] * 5), field2.shape)
def test_shapefile_through_operations(self): path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp') rd = RequestDataset(path) field = rd.get() ops = OcgOperations(dataset=rd, output_format='shp') ret = ops.execute() rd2 = RequestDataset(ret) field2 = rd2.get() self.assertAsSetEqual( list(field.keys()) + [HeaderName.ID_GEOMETRY], list(field2.keys())) self.assertEqual((51, ), field2.data_variables[0].shape)
def test_write_variable_collection_netcdf4_mpi(self): # TODO: TEST: Test writing a grouped netCDF file in parallel. self.add_barrier = False if not env.USE_NETCDF4_MPI: raise SkipTest('not env.USE_NETCDF4_MPI') path = self.create_rank_valued_netcdf() # if vm.rank == 0: # self.ncdump(path, header_only=False) rd = RequestDataset(path, driver='netcdf') rd.metadata['dimensions']['dist_dim']['dist'] = True field = rd.get() # self.barrier_print(field['data'].get_value()) if vm.rank == 0: actual_path = self.get_temporary_file_path('actual_mpi.nc') else: actual_path = None actual_path = vm.bcast(actual_path) # self.barrier_print('before field.write') field.write(actual_path) # self.barrier_print('after field.write') if vm.rank == 0: # self.ncdump(actual_path, header_only=False) self.assertNcEqual(actual_path, path)
def test_load_bounds_datetime_after_slicing(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(variable=ref_test['variable'],uri=uri) field = rd.get() slced = field[:,10:130,:,4:7,100:37] self.assertEqual(slced.temporal.bounds_datetime.shape,(120,2))
def test(self): path1 = self.write_field_data('data1') path2 = self.write_field_data('data2') path3 = self.write_field_data('basis_var') time_range = [datetime(2000, 3, 1), datetime(2000, 3, 31)] rds = [RequestDataset(p, time_range=time_range) for p in [path1, path2]] mrd = MultiRequestDataset(rds) basis = RequestDataset(path3, time_range=[datetime(2000, 8, 1), datetime(2000, 8, 31)]) basis_field = basis.get() calc = [{'func': 'mfpf', 'name': 'output_mfpf', 'kwds': {'reference': ('data1', 'data2'), 'basis': basis_field}}] ops = OcgOperations(dataset=mrd, calc=calc) ret = ops.execute() actual_field = ret.get_element() actual_variables = get_variable_names(actual_field.data_variables) self.assertEqual(actual_variables, ('diff_data1_basis_var', 'diff_data2_basis_var')) sums = [v.get_value().sum() for v in actual_field.data_variables] for s in sums: self.assertAlmostEqual(s, 7.8071042497325145)
def run_system_splitting_unstructured(self, genweights): env.CLOBBER_UNITS_ON_BOUNDS = False ufile = self.get_temporary_file_path('ugrid.nc') resolution = 10. self.fixture_regular_ugrid_file(ufile, resolution) src_rd = RequestDataset(ufile, driver=DriverNetcdfUGRID, grid_abstraction='point') # src_rd.inspect() src_grid = src_rd.get().grid self.assertEqual(src_grid.abstraction, 'point') dst_grid = self.get_gridxy_global(resolution=20., crs=Spherical()) gs = GridChunker(src_grid, dst_grid, (3, 3), check_contains=False, src_grid_resolution=10., paths=self.fixture_paths, genweights=genweights, use_spatial_decomp=True) gs.write_chunks() actual = gs.create_full_path_from_template('src_template', index=1) actual = RequestDataset(actual).get() self.assertIn(GridChunkerConstants.IndexFile.NAME_SRCIDX_GUID, actual)
def test_load(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(variable=ref_test['variable'],uri=uri) field = rd.get() ds = nc.Dataset(uri,'r') self.assertEqual(field.level,None) self.assertEqual(field.spatial.crs,WGS84()) tv = field.temporal.value test_tv = ds.variables['time'][:] self.assertNumpyAll(tv,test_tv) self.assertNumpyAll(field.temporal.bounds,ds.variables['time_bnds'][:]) tdt = field.temporal.value_datetime self.assertEqual(tdt[4],dt(2001,1,5,12)) self.assertNumpyAll(field.temporal.bounds_datetime[1001],np.array([dt(2003,9,29),dt(2003,9,30)])) rv = field.temporal.value_datetime[100] rb = field.temporal.bounds_datetime[100] self.assertTrue(all([rv > rb[0],rv < rb[1]])) self.assertEqual(field.temporal.extent_datetime,(datetime.datetime(2001,1,1),datetime.datetime(2011,1,1))) ds.close()
def test_system_with_time_data(self): """Test writing data with a time dimension.""" path = self.get_temporary_file_path('what.shp') t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time') geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='time') field = Field(variables=[t, geom], dimension_map={ 'time': { 'variable': 'time' }, 'geom': { 'variable': 'geom' } }) field.write(path, iter_kwargs={'variable': 'time'}, driver=DriverVector) rd = RequestDataset(uri=path) field2 = rd.get() # netcdftime worthlessness poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'], ['1-01-02 12:00:00', '1-01-03 12:00:00']] actual = field2['TIME'].get_value().tolist() res = [p == actual for p in poss] self.assertTrue(any(res))
def test_get_field_multifile_load(self): uri = self.test_data.get_uri('narccap_pr_wrfg_ncep') rd = RequestDataset(uri, 'pr') field = rd.get() self.assertEqual(field.temporal.extent_datetime, (datetime.datetime(1981, 1, 1, 0, 0), datetime.datetime(1991, 1, 1, 0, 0))) self.assertAlmostEqual(field.temporal.resolution, 0.125)
def test_load_time_range(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(variable=ref_test['variable'],uri=uri,time_range=[dt(2005,2,15),dt(2007,4,18)]) field = rd.get() self.assertEqual(field.temporal.value_datetime[0],dt(2005, 2, 15, 12, 0)) self.assertEqual(field.temporal.value_datetime[-1],dt(2007, 4, 18, 12, 0)) self.assertEqual(field.shape,(1,793,1,64,128))
def test_system_mftime(self): """Test a multi-file dataset with varying units on the time variables.""" paths = create_mftime_nc_files(self, with_all_cf=True) rd = RequestDataset(paths) field = rd.get() self.assertIsNone(field.temporal._value) desired = [0., 1., 2., 366., 367., 368.] actual = field.temporal.get_value().tolist() self.assertEqual(actual, desired) ops = OcgOperations(dataset=rd, output_format=constants.OutputFormatName.NETCDF) ret = ops.execute() out_rd = RequestDataset(uri=ret) self.assertEqual(out_rd.get().temporal.get_value().tolist(), desired)
def test_get_intersects(self): # test with vector geometries path = os.path.join(self.path_bin, 'shp', 'state_boundaries', 'state_boundaries.shp') rd = RequestDataset(path) field = rd.get() polygon = wkb.loads( '\x01\x06\x00\x00\x00\x03\x00\x00\x00\x01\x03\x00\x00\x00\x01\x00\x00\x00Y\x00\x00\x00\x06]\xcf\xdfo\xd4Q\xc0AV:L\xd7\xe2D@i\x856A\xbf\xd5Q\xc0^`\xe7\x0ch\xe4D@\xd8\xcb\xd4e\x1c\xd6Q\xc0\xf7\x8a\xf1\xab\x15\xe8D@*}\xf3#i\xd5Q\xc0\xda\x02\x0b\xc7\xcf\xedD@P\xbd\xdch\xeb\xd5Q\xc0R\xacZ\xab\x19\xf0D@Hd\x0bIQ\xd5Q\xc0K\x9e\xe3\'\xb1\xf2D@\xd4\xcd\xb4\xb0\x92\xd8Q\xc0^\xbf\x93a\xb8\xf1D@{u\xecSy\xd8Q\xc0o\xc6\x82\x80X\xfdD@\x80t%\xb5;\xd8Q\xc0\x84\x84\x0e\\\xc1\x01E@)\xe1\xbd\xe5\xd5\xdfQ\xc0Jp\xdd6/\x01E@\xf2\xd9\xdb\xaa\x0f\xf3Q\xc0dX\xfc\x0f\x8c\x00E@\x83\xbd\xf9\x8aY\xf3Q\xc0\x1e\xcd\x14\x15M\x02E@[\xbbY\x02\x14\x06R\xc0Z\xe9\xc5dM\x03E@\xc1\xbd\xad\xe5\xb9\x08R\xc0p\x8d\x1a\'a\x03E@\xfbw`\x10| R\xc0z3\xfd&\xf0\x03E@\xe4\x98\x9a\xf8\x8e$R\xc0r.\xe4%\xdb\x03E@\xb2\x07\xf7\xf7=%R\xc0cs\xba\x07\xc4\x02E@\x81\xa6\xef\x9b\xe6&R\xc0\xf8qV\x1f\xeb\x02E@\xa6~rz\x02\'R\xc0*\xf6\x9b\x9d\xe8\x03E@\xb3\xefU\x92`0R\xc0\xa4SJ\x1cU\x04E@\xb41\x00\xf4\x1f1R\xc0tK0\x05G\x00E@\r\x98h\xdbT4R\xc0\x1c\xc0$\xc5\xa3\xffD@\x03\xa2\xcd\xbc@4R\xc0\xc0\xe2)\xf8I\x04E@\x0cKd\xddc@R\xc0`\xfau\xf4\x9b\x04E@\x03\xd4\x96\xa3\xebBR\xc0\x82\x8en\xd1\xa5\x04E@\x1e\xca\xef\xa0\xfd^R\xc0\x02\xc7\xf9!\x12\x06E@P\xda\xb7\xff\xec_R\xc0\x12\xc1\xa68\xea\tE@2\xe2\x9d\xe7sVR\xc0\x1e\xfb\xe8\xd2\x9b@E@\xb2:3\x0f\x84PR\xc0H-0\xd7~_E@\xf8\x1c\xed\xafBAR\xc0&\xc0\xe3N\xc5^E@\x8c\x1e\x1ec\x12;R\xc0\xe4R\xa1\xf4a^E@5s\nW+\x1dR\xc0\x84\x8b\xf9\xba\xe8\\E@^L\x19*\xea\x11R\xc0\xf8\x16RF8\\E@\x9eZ\xcb\xa9\x88\xfbQ\xc0\xa8\xdb\'\xd6\x85ZE@?+\xbd\t\xa9\xf9Q\xc0~K\x9d\xd6IZE@\x0f\x8f\x0bda\xd2Q\xc0\xee\x90\xcb\xd5kYE@\xec\xa8\x91\x81\'\xd0Q\xc0?\x7fM\xd7\xef\\E@64"\x03d\xcfQ\xc0\xd6\x99\x80\xd2,_E@\xa9\xbb\x11\x1d\xed\xcbQ\xc0\xca\x7f \xb3\x8f^E@4r\xfa\x81\x96\xcbQ\xc0\x9c\x17\xed,VgE@\x9d\xee\xf0\xfa\xb7\xc7Q\xc0\xb2\xd4\x9fq\xbdhE@S\xe2r42\xc4Q\xc0\n\x1c\xe1\xef\xf3fE@!\x83y\x95\xa0\xc1Q\xc0O\x14\xf1.\xf3lE@\xb0\xf2^,\xf7\xbaQ\xc0\xfe\x89\x10\x93LqE@\xf6\x0f\xa9\xa7z\xb9Q\xc0FI\x942\x85qE@\xca>\xfb$b\xb6Q\xc0:\xa9\x7f\xda\x84nE@\x99f=\x9d\x16\xb4Q\xc0b6z\xff\xfbnE@\xf8\x1c\xcc*W\xafQ\xc0\xc8\x1fmU\xeeTE@\xcc\xb7\t\xfa\xf6\xa5Q\xc0p\xa3_"\xbaRE@Qf[{\x8a\xa8Q\xc0J\x07l\x06\x94JE@b\x8c\x1fK\n\xb4Q\xc0 \xffz\xa0\xf1EE@6\xa8\xee\xcf0\xb9Q\xc0:P\xe3MZ9E@\xf1H\xcc\xd5z\xbdQ\xc0\x82\x19u\xaaX7E@n\x18\xea\xb6/\xc2Q\xc0\x175gx\x8f$E@U\xadT\xc7\x15\xbbQ\xc0J_B\xaa\x04\x1eE@\xcb\x88\xa5\x86!\xb9Q\xc0z\xf3\xde\xa1\x04"E@\x1f\x04\x08@\xc7\xb4Q\xc0\x8co.NX!E@?\xce\x01\xf8\x92\xb1Q\xc0\x10\xf6\x91r\xd3\x1fE@nD\xd5\x08\xe8\xabQ\xc0\n\xf6\x9b\xf4\x9a\x13E@\x069\x91\xd5\x98\xa7Q\xc0\xc4\xd5\x10\xa1\xed\xfbD@L\x1b\xef\xe6\x94\xa2Q\xc0$\xad\x14j)\xf7D@\xa4\xe1T\xc3i\xa2Q\xc09\x04\xa28#\xe7D@\xd8\xea\xfa\xce\x1a\x9bQ\xc0g\x01\x88\x04/\xdfD@\xde\xc6#\x80\x86\x91Q\xc0\xff\x88\x16w_\xdcD@\x14hp\x07\xd5\x95Q\xc0(\x95L\xb3\x1c\xdbD@\xa1 \xbe\xf7"\x8dQ\xc0\x19(\xa4\x9a5\xdbD@\x9bKt\xce:\x81Q\xc0\xb8\xe3\x9b\xd3\x08\xe4D@\x175%X\x07\x80Q\xc0\x16\xe7\x88\xe3\x9c\xedD@\x8b\x0e\x11\x8cn\x86Q\xc0pZ\xae\xe7G\x00E@\xef\x08`YT\x90Q\xc0\x90\xc7\xcc\xfd\xb1\x07E@P\x02\xa0Q\xa5\x88Q\xc0\xc2\xf2\xd2~G\tE@\xf4\x84\xd0\xeb:\x83Q\xc0\xbd@\xb0\xbe]\x03E@\xb0G/\xf7\xb4}Q\xc0\xcdI<]\xb9\xf3D@\xae\x04l\xe9\xbczQ\xc0@\x99+wB\xe2D@5\xb6ME\x15}Q\xc0,\xc8f\x8f\xf3\xd5D@\xe6[z\x8br\x99Q\xc0x0\x10\xbdh\xceD@\xc8\x01\xfe\xf2\xb4\x9bQ\xc0\xf0\xb6\xcf\xc6\xed\xc8D@*\x82\xc1\xe3\xc6\xa8Q\xc0\x06n9O\x18\xc5D@\x0ei\x7f\x87\x8d\xaaQ\xc0z\x0cy./\xc7D@\xd8\x12$+\xaa\xa7Q\xc0\x9b\x93\x1bU)\xdeD@\x8f\x07\xb59\xb9\xb5Q\xc0\x9c\xcf\x98t7\xd0D@/\xb9#\xa1\x18\xb9Q\xc0\x94\xc3X\n$\xd1D@\xb2\xeeYk\x13\xc0Q\xc0\xa22ko\x93\xc2D@\xd0PQ\x18\x7f\xc7Q\xc0W\xa9\xe8\xaa\x1c\xbfD@0v(\x9f\t\xc9Q\xc0\xfa\x02g\xff\xdf\xd3D@H\x9dJF\xb9\xccQ\xc0\x0c\xc0\x99\x19\xd9\xd6D@\xb1\xfd\r\x8c\xa7\xceQ\xc0\xa4m\x9f\xba\x95\xdaD@\x96/\xfdo\x10\xd1Q\xc0Xm3\x97\xf7\xdfD@\x06]\xcf\xdfo\xd4Q\xc0AV:L\xd7\xe2D@\x01\x03\x00\x00\x00\x01\x00\x00\x00\x0f\x00\x00\x00\xc0\xb6\x07]\xad\xa6Q\xc02\xbc\x8c5\xff\xb6D@\xa9xP\x1aU\xa4Q\xc02\x92"\xe9v\xbbD@\xf2\xb9_\x96a\xa3Q\xc0o\xeeb\xfbl\xb5D@"\xfdj\xd8\xda\xa4Q\xc0\xc0\x90\x1a;\x84\xb4D@\x186*V\xf8\xa0Q\xc0Z"\x89M\x07\xb3D@\xc7j=\xf0\x1c\x9fQ\xc0\xed3hH\xb8\xabD@\x881\xcdxF\xafQ\xc0zJ`\x9a\xc5\xaaD@L7j\xfbB\xb1Q\xc0h>\xfc?*\xa6D@\xe3\xd2!\xca\x02\xb6Q\xc0\xac!n\xe7\x9e\xacD@p\xbc\xa4\xe0\x14\xb2Q\xc0\x7f\x12\xffI\x1f\xadD@\xbb\x0c\x1b\xdbV\xb1Q\xc0\\\xcd\xe5\xf4\x98\xa9D@\x80\xe8\xd2\xfc\x1c\xb0Q\xc0\x14C\x00\xed\xea\xb0D@\xc7\'\xb0 \xb8\xaaQ\xc0W\x81:c;\xbaD@\x86\x9c\x9f\x1e\xc6\xa6Q\xc02\xfc\xe8\xc4\xc1\xbcD@\xc0\xb6\x07]\xad\xa6Q\xc02\xbc\x8c5\xff\xb6D@\x01\x03\x00\x00\x00\x01\x00\x00\x00\r\x00\x00\x00k\x97\xa4\xa3\x07\x82Q\xc0@\xa7\xf3]\xed\xa7D@\xb8\xaa\xa0\xa1j\x80Q\xc0\x98+\xd84\x92\xa9D@T@x%\xb4\x81Q\xc0x\xd7\x92\xb5)\xabD@R8\x8a\xc8\x9b\x85Q\xc0\xa8\xc2\x93 \xff\xa5D@]r\xdd\x055\x82Q\xc0\xfcUH\x92\xc3\xacD@\xf7"J%\'\x83Q\xc0\xb0)@\xca+\xb2D@\xb8\xfb\xdf\x9e\xd2}Q\xc0\xe1A\x12\x00\xbf\xa5D@\x0f\xd7\xa3\xfd\xfa}Q\xc0\x99\x08\xc8\x84;\xa0D@\x85\xbc\xcfF\x99\x86Q\xc0\x10\xdd1\xf0\x7f\x9eD@eg\xec/\xa6\x8dQ\xc0\x99\xdf\xe4\x16\x96\xa2D@\'\xdc\xad\x10A\x8dQ\xc0\x1ag\xa1\xa7\xa4\xa5D@$\xc4\x04\x8aC\x86Q\xc0Uu\xb2l\x89\xa3D@k\x97\xa4\xa3\x07\x82Q\xc0@\xa7\xf3]\xed\xa7D@') ret = field.get_intersects(polygon) self.assertTrue(polygon.almost_equals(ret.spatial.geom.polygon.value.compressed()[0]))
def get_large_vector(): uri = '/home/ben.koziol/ocgis_test_data/nc/CanCM4/tas_day_CanCM4_decadal2000_r2i1p1_20010101-20101231.nc' rd = RequestDataset(uri=uri) field = rd.get() print field.shape with open('/tmp/my_coords.txt', 'w') as f: f.write(str(field.spatial.grid.value[1].flatten().tolist())) f.write('/n') f.write(str(field.spatial.grid.value[0].flatten().tolist()))
def test_load_projection_axes(self): uri = self.test_data.get_uri('cmip3_extraction') variable = 'Tavg' rd = RequestDataset(uri,variable) with self.assertRaises(DimensionNotFound): rd.get() rd = RequestDataset(uri,variable,dimension_map={'R':'projection','T':'time','X':'longitude','Y':'latitude'}) field = rd.get() self.assertEqual(field.shape,(36, 1800, 1, 7, 12)) self.assertEqual(field.temporal.value_datetime[0],datetime.datetime(1950, 1, 16, 0, 0)) self.assertEqual(field.temporal.value_datetime[-1],datetime.datetime(2099, 12, 15, 0, 0)) self.assertEqual(field.level,None) self.assertNumpyAll(field.realization.value,np.arange(1,37)) ds = nc.Dataset(uri,'r') to_test = ds.variables['Tavg'] self.assertNumpyAll(to_test[:],field.variables['Tavg'].value.squeeze()) ds.close()
def test_load_climatology_bounds(self): rd = self.test_data.get_rd('cancm4_tas') ops = ocgis.OcgOperations(dataset=rd,output_format='nc',geom='state_boundaries', select_ugid=[27],calc=[{'func':'mean','name':'mean'}], calc_grouping=['month']) ret = ops.execute() rd = RequestDataset(uri=ret,variable='mean') field = rd.get() self.assertNotEqual(field.temporal.bounds,None)
def test_esmf(self): rd1 = RequestDataset(**self.get_dataset()) rd2 = deepcopy(rd1) ops = OcgOperations(dataset=rd1, regrid_destination=rd2, output_format='nc') ret = ops.execute() actual_value = RequestDataset(ret).get().data_variables[0].get_value() desired_value = rd1.get().data_variables[0].get_value() self.assertNumpyAllClose(actual_value, desired_value)
def test_get_field_t_conform_units_to(self): """ Test conforming time units is appropriately passed to field object. """ uri = self.test_data.get_uri('cancm4_tas') target = get_units_object('days since 1949-1-1', calendar='365_day') rd = RequestDataset(uri=uri, t_conform_units_to=target) field = rd.get() self.assertEqual(field.temporal.conform_units_to, target)
def test_load_time_region_slicing(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo', time_region={'month':[1,10],'year':[2011,2013]}) with self.assertRaises(EmptySubsetError): rd.get() rd = RequestDataset(variable=ref_test['variable'],uri=uri,alias='foo', time_region={'month':[1,10],'year':[2005,2007]}) field = rd.get() sub = field[:,:,:,50,75] self.assertEqual(sub.shape,(1,124,1,1,1)) self.assertEqual(sub.variables['foo'].value.shape,(1,124,1,1,1)) field = rd.get() sub = field[:,:,:,50,75:77] sub2 = field[:,:,:,0,1] self.assertEqual(sub2.shape,(1, 124, 1, 1, 1))
def test_get_dimension_map_with_spatial_mask(self): path = self.get_temporary_file_path('foo.nc') grid = create_gridxy_global() gmask = grid.get_mask(create=True) gmask[1, 1] = True grid.set_mask(gmask) grid.parent.write(path) rd = RequestDataset(path) driver = DriverNetcdfCF(rd) dmap = driver.get_dimension_map(driver.metadata_source) self.assertIsNotNone(dmap.get_spatial_mask()) field = rd.get() self.assertEqual(field.grid.get_mask().sum(), 1) # Test mask variable is blown away if set to None during a read. rd = RequestDataset(path) rd.dimension_map.set_spatial_mask(None) self.assertIsNone(rd.dimension_map.get_spatial_mask()) # rd.dimension_map.pprint() field = rd.get() self.assertIsNone(field.grid.get_mask())
def test_load_datetime_slicing(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(variable=ref_test['variable'],uri=uri) field = rd.get() field.temporal.value_datetime field.temporal.bounds_datetime slced = field[:,239,:,:,:] self.assertEqual(slced.temporal.value_datetime,np.array([dt(2001,8,28,12)])) self.assertNumpyAll(slced.temporal.bounds_datetime,np.array([dt(2001,8,28),dt(2001,8,29)]))
def test_load_projection_axes_slicing(self): uri = self.test_data.get_uri('cmip3_extraction') variable = 'Tavg' rd = RequestDataset(uri,variable,dimension_map={'R':'projection','T':'time','X':'longitude','Y':'latitude'}) field = rd.get() sub = field[15,:,:,:,:] self.assertEqual(sub.shape,(1,1800,1,7,12)) ds = nc.Dataset(uri,'r') to_test = ds.variables['Tavg'] self.assertNumpyAll(to_test[15,:,:,:],sub.variables[variable].value.squeeze()) ds.close()
def test_create_dimension_map_with_spatial_mask(self): path = self.get_temporary_file_path('foo.nc') grid = create_gridxy_global() gmask = grid.get_mask(create=True) gmask[1, 1] = True grid.set_mask(gmask) grid.parent.write(path) rd = RequestDataset(path) driver = DriverNetcdfCF(rd) dmap = driver.create_dimension_map(driver.metadata_source) self.assertIsNotNone(dmap.get_spatial_mask()) field = rd.get() self.assertEqual(field.grid.get_mask().sum(), 1) # Test mask variable is blown away if set to None during a read. rd = RequestDataset(path) rd.dimension_map.set_spatial_mask(None) self.assertIsNone(rd.dimension_map.get_spatial_mask()) # rd.dimension_map.pprint() field = rd.get() self.assertIsNone(field.grid.get_mask())
def test_crs_with_dimension_map(self): """Test CRS overloading in the presence of a dimension map.""" field = self.get_field() path = self.get_temporary_file_path('foo.nc') field.write(path) dmap = {DimensionMapKey.X: {DimensionMapKey.VARIABLE: 'col'}, DimensionMapKey.Y: {DimensionMapKey.VARIABLE: 'row'}} rd = RequestDataset(path, dimension_map=dmap, crs=Tripole()) field = rd.get() self.assertIsInstance(field.crs, Tripole)
def get_ofield(self): rd = RequestDataset(**self.get_dataset()) # Create a field composed of two variables. ofield = rd.get() new_variable = ofield['foo'].extract() new_variable.set_name('foo2') new_variable.get_value()[:] = 30 ofield.add_variable(new_variable, is_data=True) ofield = ofield.get_field_slice({'level': 0}) return ofield
def test_system_spatial_averaging_from_file(self): rd_nc = self.test_data.get_rd('cancm4_tas') rd_shp = RequestDataset(self.path_state_boundaries) field_shp = rd_shp.get() actual = field_shp.dimension_map.get_variable(DMK.GEOM) self.assertIsNotNone(actual) actual = field_shp.dimension_map.get_dimension(DMK.GEOM) self.assertEqual(len(actual), 1) self.assertEqual(field_shp.crs, WGS84()) try: index_geom = np.where( field_shp['STATE_NAME'].get_value() == 'Nebraska')[0][0] except IndexError: # Not found on rank. polygon_field = None else: polygon_field = field_shp.get_field_slice({'geom': index_geom}) polygon_field = MPI_COMM.gather(polygon_field) if MPI_RANK == 0: for p in polygon_field: if p is not None: polygon_field = p break polygon_field = MPI_COMM.bcast(polygon_field) polygon_field.unwrap() polygon = polygon_field.geom.get_value()[0] field_nc = rd_nc.get() sub_field_nc = field_nc.get_field_slice({'time': slice(0, 10)}) self.assertEqual(sub_field_nc['tas']._dimensions, field_nc['tas']._dimensions) sub = sub_field_nc.grid.get_intersects(polygon) # When split across two processes, there are floating point summing differences. desired = {1: 2734.5195, 2: 2740.4014} with vm.scoped_by_emptyable('grid intersects', sub): if not vm.is_null: abstraction_geometry = sub.get_abstraction_geometry() sub.parent.add_variable(abstraction_geometry, force=True) unioned = abstraction_geometry.get_unioned( spatial_average='tas') if unioned is not None: tas = unioned.parent['tas'] self.assertFalse(tas.is_empty) self.assertAlmostEqual(tas.get_value().sum(), desired[vm.size], places=4)
def test_get_esmf_grid_bilinear_regrid_method(self): """Test with a regrid method that does not require corners.""" from ocgis.regrid.base import get_esmf_grid from ESMF import RegridMethod import ESMF rd = RequestDataset(**self.get_dataset()) field = rd.get() self.assertTrue(field.grid.has_bounds) egrid = get_esmf_grid(field.grid, regrid_method=RegridMethod.BILINEAR) corner = egrid.coords[ESMF.StaggerLoc.CORNER] for idx in [0, 1]: self.assertIsNone(corner[idx])
def test_system_renaming_dimensions_on_variables(self): var1 = Variable(name='var1', value=[1, 2], dimensions='dim') var2 = Variable(name='var2', value=[3, 4], dimensions='dim') vc = VariableCollection(variables=[var1, var2]) path = self.get_temporary_file_path('out.nc') vc.write(path) rd = RequestDataset(path) meta = rd.metadata meta['dimensions']['new_dim'] = {'size': 2} meta['variables']['var2']['dimensions'] = ('new_dim',) field = rd.get() self.assertEqual(field['var2'].dimensions[0].name, 'new_dim') self.assertEqual(field['var2'].get_value().tolist(), [3, 4]) path2 = self.get_temporary_file_path('out2.nc') field.write(path2) rd2 = RequestDataset(path2) field2 = rd2.get() self.assertEqual(field2['var2'].dimensions[0].name, 'new_dim') self.assertEqual(field2['var2'].get_value().tolist(), [3, 4])
def test_write_variable_collection_parallel(self): if MPI_RANK == 0: path1 = self.get_temporary_file_path('out1.shp') path2 = self.get_temporary_file_path('out2.shp') else: path1, path2 = [None] * 2 path1 = MPI_COMM.bcast(path1) path2 = MPI_COMM.bcast(path2) # Test writing the field to file. driver = self.get_driver() field = driver.create_field() # Only test open file objects on a single processor. if MPI_SIZE == 1: fiona_crs = get_fiona_crs(field) fiona_schema = get_fiona_schema(field.geom.geom_type, six.next(field.iter())[1]) fobject = fiona.open(path2, mode='w', schema=fiona_schema, crs=fiona_crs, driver='ESRI Shapefile') else: fobject = None for target in [path1, fobject]: # Skip the open file object test during a multi-proc test. if MPI_SIZE > 1 and target is None: continue field.write(target, driver=DriverVector) if isinstance(target, six.string_types): path = path1 else: path = path2 fobject.close() if MPI_RANK == 0: with fiona.open(path) as source: self.assertEqual(len(source), 51) rd = RequestDataset(uri=path) field2 = rd.get() for v in list(field.values()): if isinstance(v, CoordinateReferenceSystem): self.assertEqual(v, field2.crs) else: self.assertNumpyAll(v.get_value(), field2[v.name].get_value())
def test_get_field_with_projection(self): uri = self.test_data.get_uri('narccap_wrfg') rd = RequestDataset(uri, 'pr') field = rd.get() self.assertIsInstance(field.spatial.crs, CFLambertConformal) field.spatial.update_crs(CFWGS84()) self.assertIsInstance(field.spatial.crs, CFWGS84) self.assertEqual(field.spatial.grid.row, None) self.assertAlmostEqual(field.spatial.grid.value.mean(), -26.269666952512416) field.spatial.crs.unwrap(field.spatial) self.assertAlmostEqual(field.spatial.grid.value.mean(), 153.73033304748759) self.assertIsNone(field.spatial.geom.polygon) self.assertAlmostEqual(field.spatial.geom.point.value[0, 100].x, 278.52630062012787) self.assertAlmostEqual(field.spatial.geom.point.value[0, 100].y, 21.4615681252577)
def test_get_esmf_grid_bilinear_regrid_method(self): """Test with a regrid method that does not require corners.""" from ocgis.regrid.base import create_esmf_grid from ESMF import RegridMethod import ESMF rd = RequestDataset(**self.get_dataset()) field = rd.get() self.assertTrue(field.grid.has_bounds) egrid = create_esmf_grid(field.grid, regrid_method=RegridMethod.BILINEAR) corner = egrid.coords[ESMF.StaggerLoc.CORNER] for idx in [0, 1]: self.assertIsNone(corner[idx])
def test_get_field(self): ref_test = self.test_data['cancm4_tas'] uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(variable=ref_test['variable'], uri=uri) field = rd.get() self.assertIsInstance(field.spatial.grid, NcSpatialGridDimension) # test names are correctly set when creating the field self.assertEqual(field.temporal.name, 'time') self.assertEqual(field.temporal.name_value, 'time') self.assertEqual(field.temporal.name_bounds, 'time_bnds') row = field.spatial.grid.row self.assertEqual(row.name, 'lat') self.assertEqual(row.name_value, 'lat') self.assertEqual(row.name_bounds, 'lat_bnds') col = field.spatial.grid.col self.assertEqual(col.name, 'lon') self.assertEqual(col.name_value, 'lon') self.assertEqual(col.name_bounds, 'lon_bnds') # test attributes are loaded self.assertEqual(len(field.attrs), 31) self.assertEqual(len(field.variables['tas'].attrs), 10) self.assertEqual(len(field.temporal.attrs), 6) self.assertEqual(len(field.spatial.grid.row.attrs), 5) self.assertEqual(len(field.spatial.grid.col.attrs), 5) ds = nc.Dataset(uri, 'r') self.assertEqual(field.level, None) self.assertEqual(field.spatial.crs, WGS84()) tv = field.temporal.value test_tv = ds.variables['time'][:] self.assertNumpyAll(tv, test_tv) self.assertNumpyAll(field.temporal.bounds, ds.variables['time_bnds'][:]) tdt = field.temporal.value_datetime self.assertEqual(tdt[4], dt(2001, 1, 5, 12)) self.assertNumpyAll(field.temporal.bounds_datetime[1001], np.array([dt(2003, 9, 29), dt(2003, 9, 30)])) rv = field.temporal.value_datetime[100] rb = field.temporal.bounds_datetime[100] self.assertTrue(all([rv > rb[0], rv < rb[1]])) self.assertEqual(field.temporal.extent_datetime, (datetime.datetime(2001, 1, 1), datetime.datetime(2011, 1, 1))) ds.close()
def test_get_variable_value(self): path1 = self.get_temporary_file_path('f1.nc') path2 = self.get_temporary_file_path('f2.nc') tdim = Dimension('time') t1 = TemporalVariable(value=[1, 2, 3, 4, 5], dimensions=tdim, dtype=np.int32) k = {KeywordArgument.DATASET_KWARGS: {KeywordArgument.FORMAT: 'NETCDF4_CLASSIC'}} t1.parent.write(path1, **k) t2 = TemporalVariable(value=[6, 7, 8, 9, 10], dimensions=tdim, dtype=np.int32) t2.parent.write(path2, **k) rd = RequestDataset([path1, path2]) tin = rd.get().time sub = tin[2:8] self.assertEqual(sub.get_value().tolist(), list(range(3, 9)))
def test_init(self): # Test passing an open dataset object. rd = self.get_request_dataset_netcdf() path = rd.uri with self.nc_scope(path) as ds: for _ in range(1): rd2 = RequestDataset(opened=ds, driver=DriverNetcdf) field = rd2.get() self.assertIsInstance(field, Field) # Test unique identifier. rd = self.get_request_dataset_netcdf(uid=43) self.assertEqual(rd.uid, 43) field = rd.get() self.assertEqual(field.uid, 43)
def test_system_parallel_write_ndvariable(self): """Test a parallel vector GIS write with a n-dimensional variable.""" ompi = OcgDist() ompi.create_dimension('time', 3) ompi.create_dimension('extra', 2) ompi.create_dimension('x', 4) ompi.create_dimension('y', 7, dist=True) ompi.update_dimension_bounds() if MPI_RANK == 0: path = self.get_temporary_file_path('foo.shp') t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time') t.set_extrapolated_bounds('the_time_bounds', 'bounds') extra = Variable(name='extra', value=[7, 8], dimensions='extra') x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float) x.set_extrapolated_bounds('x_bounds', 'bounds') # This will have the distributed dimension. y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float) y.set_extrapolated_bounds('y_bounds', 'bounds') data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x']) dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'}, 'y': {'variable': 'y', 'bounds': 'y_bounds'}, 'time': {'variable': 'time', 'bounds': 'the_time_bounds'}} vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data') vc.set_abstraction_geom() else: path, vc = [None] * 2 path = MPI_COMM.bcast(path) vc = variable_collection_scatter(vc, ompi) with vm.scoped_by_emptyable('write', vc): if not vm.is_null: vc.write(path, driver=DriverVector) MPI_COMM.Barrier() desired = 168 rd = RequestDataset(path, driver=DriverVector) sizes = MPI_COMM.gather(rd.get().geom.shape[0]) if MPI_RANK == 0: self.assertEqual(sum(sizes), desired)
def test_get_dimension_map_2d_spatial_coordinates(self): grid = create_gridxy_global() grid.expand() path = self.get_temporary_file_path('foo.nc') f = Field(grid=grid) f.write(path) rd = RequestDataset(path) field = rd.get() sub = field.get_field_slice({'y': 10, 'x': 5}) self.assertEqual(sub.grid.x.shape, (1, 1)) actual = f.dimension_map.get_dimension(DimensionMapKey.Y) self.assertEqual(actual, ['y']) actual = f.dimension_map.get_dimension(DimensionMapKey.X) self.assertEqual(actual, ['x'])
def test_create_dimension_map_2d_spatial_coordinates(self): grid = create_gridxy_global() grid.expand() path = self.get_temporary_file_path('foo.nc') f = Field(grid=grid) f.write(path) rd = RequestDataset(path) field = rd.get() sub = field.get_field_slice({'y': 10, 'x': 5}) self.assertEqual(sub.grid.x.shape, (1, 1)) actual = f.dimension_map.get_dimension(DimensionMapKey.Y) self.assertEqual(actual, ['y']) actual = f.dimension_map.get_dimension(DimensionMapKey.X) self.assertEqual(actual, ['x'])
def test(self): path = self.get_path_to_template_csv() path_out = self.get_temporary_file_path('foo_out.csv') rd = RequestDataset(uri=path) vc = rd.create_raw_field() for v in list(vc.values()): self.assertIsNotNone(v.get_value()) field = rd.get() self.assertIsInstance(field, Field) field.write(path_out, driver=DriverCSV) self.assertCSVFilesEqual(path, path_out)
def test_system_add_variable(self): """Test adding variables from spatial collections.""" # Create a few separate fields. variable_names = tuple(['a', 'b', 'c']) fields = [self.get_field(variable_name=v) for v in variable_names] # Create spatial collections containing those fields. scs = [] for field in fields: sc = SpatialCollection() sc.add_field(field, None) scs.append(sc) # Destination spatial collection to add variables to from source spatial collections. grow = scs[0] # Loop over source fields. for idx in range(1, len(scs)): # Loop over child fields and spatial containers in the current source spatial collection. for field, container in scs[idx].iter_fields(yield_container=True): # TODO: This should be adjusted to allow easier selection with empty fields. try: # Case when we have spatial containers. grow_field = grow.get_element(field_name=field.name, container_ugid=container) except KeyError: # Case without spatial containers. grow_field = grow.get_element(field.name) # Add data variables to the grow field. for dv in field.data_variables: grow_field.add_variable(dv.extract(), is_data=True) # Assert all variables are present on the grow field. actual = grow.get_element() self.assertEqual(get_variable_names(actual.data_variables), variable_names) # Write the spatial collection using a converter. conv = NcConverter([grow], outdir=self.current_dir_output, prefix='out.nc') conv.write() # Assert all variables are present. rd = RequestDataset(conv.path) actual = rd.get() self.assertEqual(get_variable_names(actual.data_variables), variable_names)
def test_get_esmf_grid(self): import ESMF rd = RequestDataset(**self.get_dataset()) have_ocgis_bounds = [True, False] for h in have_ocgis_bounds: field = rd.get() ogrid = field.grid if not h: ogrid.remove_bounds() self.assertFalse(ogrid.has_bounds) from ocgis.regrid.base import get_esmf_grid egrid = get_esmf_grid(ogrid) # ocgis is row major with esmf being column major (i.e. in ocgis rows are stored in the zero index) for idx_esmf, idx_ocgis in zip([0, 1], [1, 0]): coords = egrid.coords[ESMF.StaggerLoc.CENTER][idx_esmf] desired = ogrid.get_value_stacked()[idx_ocgis, :] self.assertNumpyAll(coords, desired) corner = egrid.coords[ESMF.StaggerLoc.CORNER] if h: corner_row = corner[1] corner_row_actual = np.array([[36.5, 36.5, 36.5, 36.5, 36.5], [37.5, 37.5, 37.5, 37.5, 37.5], [38.5, 38.5, 38.5, 38.5, 38.5], [39.5, 39.5, 39.5, 39.5, 39.5], [40.5, 40.5, 40.5, 40.5, 40.5]], dtype=ogrid.archetype.dtype) self.assertNumpyAll(corner_row, corner_row_actual) corner = egrid.coords[ESMF.StaggerLoc.CORNER] corner_col = corner[0] corner_col_actual = np.array( [[-105.5, -104.5, -103.5, -102.5, -101.5], [-105.5, -104.5, -103.5, -102.5, -101.5], [-105.5, -104.5, -103.5, -102.5, -101.5], [-105.5, -104.5, -103.5, -102.5, -101.5], [-105.5, -104.5, -103.5, -102.5, -101.5]], dtype=ogrid.archetype.dtype) self.assertNumpyAll(corner_col, corner_col_actual) else: # No corners should be on the ESMF grid. for idx in [0, 1]: self.assertIsNone(corner[idx])
def test_get_ocgis_grid_from_esmf_grid(self): from ocgis.regrid.base import get_esmf_grid from ocgis.regrid.base import get_ocgis_grid_from_esmf_grid rd = RequestDataset(**self.get_dataset()) keywords = dict(has_corners=[True, False], has_mask=[True, False], crs=[None, CoordinateReferenceSystem(epsg=4326)]) for k in itr_products_keywords(keywords, as_namedtuple=True): field = rd.get() grid = field.grid if k.has_mask: gmask = grid.get_mask(create=True) gmask[1, :] = True grid.set_mask(gmask) if not k.has_corners: grid.x.set_bounds(None) grid.y.set_bounds(None) self.assertFalse(grid.has_bounds) egrid = get_esmf_grid(grid) ogrid = get_ocgis_grid_from_esmf_grid(egrid, crs=k.crs) if k.has_mask: actual_mask = ogrid.get_mask() self.assertEqual(actual_mask.sum(), 4) self.assertTrue(actual_mask[1, :].all()) self.assertEqual(ogrid.crs, k.crs) self.assertNumpyAll(grid.get_value_stacked(), ogrid.get_value_stacked()) if k.has_corners: desired = grid.x.bounds.get_value() actual = ogrid.x.bounds.get_value() self.assertNumpyAll(actual, desired) desired = grid.y.bounds.get_value() actual = ogrid.y.bounds.get_value() self.assertNumpyAll(actual, desired) else: self.assertFalse(ogrid.has_bounds)
def test_get_esmf_grid_change_origin_row_and_col(self): """Test with different row and column grid origin.""" from ocgis.regrid.base import get_esmf_grid import ESMF rd = RequestDataset(**self.get_dataset()) field = rd.get() field.grid.y.set_value(np.flipud(field.grid.y.get_value())) field.grid.y.bounds.set_value( np.fliplr(np.flipud(field.grid.y.bounds.get_value()))) field.grid.x.set_value(np.flipud(field.grid.x.get_value())) field.grid.x.bounds.set_value( np.fliplr(np.flipud(field.grid.x.bounds.get_value()))) egrid = get_esmf_grid(field.grid) for idx_esmf, idx_ocgis in zip([0, 1], [1, 0]): coords = egrid.coords[ESMF.StaggerLoc.CENTER][idx_esmf] self.assertNumpyAll(coords, field.grid.get_value_stacked()[idx_ocgis, ...]) corner = egrid.coords[ESMF.StaggerLoc.CORNER] corner_row = corner[1] dtype = field.grid.dtype corner_row_actual = np.array( [[40.5, 40.5, 40.5, 40.5, 40.5], [39.5, 39.5, 39.5, 39.5, 39.5], [38.5, 38.5, 38.5, 38.5, 38.5], [37.5, 37.5, 37.5, 37.5, 37.5], [36.5, 36.5, 36.5, 36.5, 36.5]], dtype=dtype) self.assertNumpyAll(corner_row, corner_row_actual) corner = egrid.coords[ESMF.StaggerLoc.CORNER] corner_col = corner[0] corner_col_actual = np.array( [[-101.5, -102.5, -103.5, -104.5, -105.5], [-101.5, -102.5, -103.5, -104.5, -105.5], [-101.5, -102.5, -103.5, -104.5, -105.5], [-101.5, -102.5, -103.5, -104.5, -105.5], [-101.5, -102.5, -103.5, -104.5, -105.5]], dtype=dtype) self.assertNumpyAll(corner_col, corner_col_actual)
def test_system_predicate(self): """Test creating a request dataset with a predicate.""" path = self.get_temporary_file_path('foo.nc') field = self.get_field() to_exclude = Variable(name='exclude') field.add_variable(to_exclude) field.write(path) rd = RequestDataset(uri=path, predicate=lambda x: not x.startswith('exclude')) self.assertNotIn('exclude', rd.metadata['variables']) actual = rd.get() self.assertNotIn('exclude', actual) # Test predicate affects data variable identification. path = self.get_temporary_file_path('foo.nc') rd = RequestDataset(uri=path, predicate=lambda x: x != 'foo') with self.assertRaises(NoDataVariablesFound): assert rd.variable
def test_system_cf_data_read(self): """Test some basic reading operations.""" rd = self.test_data.get_rd('cancm4_tas') field = rd.get() self.assertIsInstance(field, Field) self.assertEqual(rd.variable, 'tas') self.assertEqual(field['tas'].units, 'K') self.assertEqual(len(field.dimensions), 4) self.assertIsNotNone(field.crs) self.assertIsInstance(field.time, TemporalVariable) # Geometry is not loaded automatically from the grid. self.assertIsNone(field.geom) field.set_abstraction_geom() self.assertIsInstance(field.geom, GeometryVariable) # Test overloading units. path = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(uri=path, units='celsius') field = rd.get() self.assertEqual(field['tas'].units, 'celsius')
def test_write_parallel(self): """Test writing by selective rank.""" if MPI_SIZE != 3 and MPI_SIZE != 1: raise SkipTest('MPI_SIZE != 1 or 3') ranks = list(range(MPI_SIZE)) for base_rank in ranks: for driver in [DriverCSV, DriverVector, DriverNetcdf]: if MPI_RANK == 0: path = self.get_temporary_file_path('{}-{}.{}'.format( driver.key, base_rank, driver.common_extension)) else: path = None path = MPI_COMM.bcast(path) with vm.scoped('field write by rank', [base_rank]): if not vm.is_null: geom = GeometryVariable( value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='geom') data = Variable(name='data', value=[10, 20], dimensions='geom') field = Field(geom=geom) field.add_variable(data, is_data=True) self.assertFalse(os.path.isdir(path)) field.write(path, driver=driver) self.assertFalse(os.path.isdir(path)) rd = RequestDataset(path, driver=driver) in_field = rd.get() self.assertEqual(in_field['data'].dimensions[0].size, 2) MPI_COMM.Barrier() MPI_COMM.Barrier()
def test_get_distributed_slice(self): with vm.scoped('grid write', [0]): if MPI_RANK == 0: x = Variable('x', list(range(768)), 'x', float) y = Variable('y', list(range(768)), 'y', float) grid = Grid(x, y) field = Field(grid=grid) path = self.get_temporary_file_path('grid.nc') field.write(path) else: path = None path = vm.bcast(path) rd = RequestDataset(path) grid = rd.get().grid bounds_global = deepcopy([d.bounds_global for d in grid.dimensions]) for _ in range(10): _ = grid.get_distributed_slice([slice(73, 157), slice(305, 386)]) bounds_global_grid_after_slice = [ d.bounds_global for d in grid.dimensions ] self.assertEqual(bounds_global, bounds_global_grid_after_slice)
def test_keyword_time_subset_func(self): def _func_(value, bounds=None): indices = [] for ii, v in enumerate(value.flat): if v.month == 6: indices.append(ii) return indices rd = self.test_data.get_rd('cancm4_tas') ops = OcgOperations(dataset=rd, time_subset_func=_func_, geom='state_boundaries', geom_select_uid=[20]) ret = ops.execute() ret = ret.get_element() for v in ret.temporal.value_datetime: self.assertEqual(v.month, 6) rd = self.test_data.get_rd('cancm4_tas') ops = OcgOperations(dataset=rd, time_subset_func=_func_, geom='state_boundaries', geom_select_uid=[20], output_format=constants.OutputFormatName.NETCDF) ret = ops.execute() rd_out = RequestDataset(ret) for v in rd_out.get().temporal.value_datetime: self.assertEqual(v.month, 6)
def test_get_intersects_one_rank_with_mask(self): """Test mask is created if one rank has a spatial mask.""" if MPI_SIZE != 2: raise SkipTest('MPI_SIZE != 2') if MPI_RANK == 0: value = [1, 2] else: value = [3, 4] ompi = OcgDist() xdim = ompi.create_dimension('x', 4, dist=True) ydim = ompi.create_dimension('y', 5, dist=False) ompi.update_dimension_bounds() x = Variable('x', value=value, dimensions=xdim) y = Variable('y', value=[1, 2, 3, 4, 5], dimensions=ydim) grid = Grid(x, y) wkt_geom = 'Polygon ((0.72993630573248502 5.22484076433120936, 0.70318471337579691 0.67707006369426814, 2.70063694267515952 0.69490445859872629, 2.59363057324840796 2.54076433121019107, 4.52866242038216527 2.51401273885350296, 4.40382165605095466 5.34968152866241908, 0.72993630573248502 5.22484076433120936))' subset_geom = wkt.loads(wkt_geom) sub = grid.get_intersects(subset_geom) path = self.get_temporary_file_path('foo.nc') field = Field(grid=sub) field.write(path) with vm.scoped('mask count', [0]): if not vm.is_null: rd = RequestDataset(path) out_field = rd.get() target = out_field[out_field.grid._mask_name].get_value() select = target != 0 self.assertEqual(select.sum(), 4)
def get_sorted_uris_by_time_dimension(uris, variable=None): """ Sort a sequence of NetCDF URIs by the maximum time extent in ascending order. :param uris: The sequence of NetCDF URIs to sort. :type uris: list[str] >>> uris = ['/path/to/file2.nc', 'path/to/file1.nc'] :param str variable: The target variable for sorting. If ``None`` is provided, then the variable will be autodiscovered. :returns: A sequence of sorted URIs. :rtype: list[str] """ from ocgis import RequestDataset to_sort = {} for uri in uris: rd = RequestDataset(uri=uri, variable=variable) to_sort[rd.get().temporal.extent_datetime[1]] = rd.uri sorted_keys = sorted(to_sort) ret = [to_sort[sk] for sk in sorted_keys] return ret
from ocgis import RequestDataset, OcgOperations from ocgis.contrib.library_icclim import IcclimTG90p ######################################################################################################################## # Compute a custom percentile basis using ICCLIM. # Path to CF climate dataset. This examples uses the same file for indice and percentile basis calculation. in_file = '/path/to/cf_data.nc' # Subset the input dataset to return the desired base period for the percentile basis. variable = 'tas' years = range(1971, 2001) time_region = {'year': years} rd = RequestDataset(uri=in_file, variable=variable) field = rd.get() field.get_time_region(time_region) # Calculate the percentile basis. The data values must be a three-dimensional array. arr = field.variables[variable].value.squeeze() dt_arr = field.temporal.value_datetime percentile = 90 window_width = 5 percentile_dict = IcclimTG90p.get_percentile_dict(arr, dt_arr, percentile, window_width) ######################################################################################################################## # Calculate indice using custom percentile basis. calc = [{ 'func': 'icclim_TG90p', 'name': 'TG90p',