def test_system_parallel_write_ndvariable(self): """Test a parallel vector GIS write with a n-dimensional variable.""" ompi = OcgDist() ompi.create_dimension('time', 3) ompi.create_dimension('extra', 2) ompi.create_dimension('x', 4) ompi.create_dimension('y', 7, dist=True) ompi.update_dimension_bounds() if MPI_RANK == 0: path = self.get_temporary_file_path('foo.shp') t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time') t.set_extrapolated_bounds('the_time_bounds', 'bounds') extra = Variable(name='extra', value=[7, 8], dimensions='extra') x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float) x.set_extrapolated_bounds('x_bounds', 'bounds') # This will have the distributed dimension. y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float) y.set_extrapolated_bounds('y_bounds', 'bounds') data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x']) dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'}, 'y': {'variable': 'y', 'bounds': 'y_bounds'}, 'time': {'variable': 'time', 'bounds': 'the_time_bounds'}} vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data') vc.set_abstraction_geom() else: path, vc = [None] * 2 path = MPI_COMM.bcast(path) vc = variable_collection_scatter(vc, ompi) with vm.scoped_by_emptyable('write', vc): if not vm.is_null: vc.write(path, driver=DriverVector) MPI_COMM.Barrier() desired = 168 rd = RequestDataset(path, driver=DriverVector) sizes = MPI_COMM.gather(rd.get().geom.shape[0]) if MPI_RANK == 0: self.assertEqual(sum(sizes), desired)
def test_system_parallel_write_ndvariable(self): """Test a parallel vector GIS write with a n-dimensional variable.""" ompi = OcgDist() ompi.create_dimension('time', 3) ompi.create_dimension('extra', 2) ompi.create_dimension('x', 4) ompi.create_dimension('y', 7, dist=True) ompi.update_dimension_bounds() if MPI_RANK == 0: path = self.get_temporary_file_path('foo.shp') t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time') t.set_extrapolated_bounds('the_time_bounds', 'bounds') extra = Variable(name='extra', value=[7, 8], dimensions='extra') x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float) x.set_extrapolated_bounds('x_bounds', 'bounds') # This will have the distributed dimension. y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float) y.set_extrapolated_bounds('y_bounds', 'bounds') data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x']) dimension_map = {'x': {'variable': 'x', 'bounds': 'x_bounds'}, 'y': {'variable': 'y', 'bounds': 'y_bounds'}, 'time': {'variable': 'time', 'bounds': 'the_time_bounds'}} vc = Field(variables=[t, extra, x, y, data], dimension_map=dimension_map, is_data='data') vc.set_abstraction_geom() else: path, vc = [None] * 2 path = MPI_COMM.bcast(path) vc = variable_collection_scatter(vc, ompi) with vm.scoped_by_emptyable('write', vc): if not vm.is_null: vc.write(path, driver=DriverVector) MPI_COMM.Barrier() desired = 168 rd = RequestDataset(path, driver=DriverVector) sizes = MPI_COMM.gather(rd.get().geom.shape[0]) if MPI_RANK == 0: self.assertEqual(sum(sizes), desired)
def test_system_with_time_data(self): """Test writing data with a time dimension.""" path = self.get_temporary_file_path('what.shp') t = TemporalVariable(value=[1.5, 2.5], name='time', dimensions='time') geom = GeometryVariable(value=[Point(1, 2), Point(3, 4)], name='geom', dimensions='time') field = Field(variables=[t, geom], dimension_map={ 'time': { 'variable': 'time' }, 'geom': { 'variable': 'geom' } }) field.write(path, iter_kwargs={'variable': 'time'}, driver=DriverVector) rd = RequestDataset(uri=path) field2 = rd.get() # netcdftime worthlessness poss = [['0001-01-02 12:00:00', '0001-01-03 12:00:00'], ['1-01-02 12:00:00', '1-01-03 12:00:00']] actual = field2['TIME'].get_value().tolist() res = [p == actual for p in poss] self.assertTrue(any(res))
def get_ocgfield_example(self): dtime = Dimension(name='time') t = TemporalVariable(value=[1, 2, 3, 4], name='the_time', dimensions=dtime, dtype=float) t.set_extrapolated_bounds('the_time_bounds', 'bounds') lon = Variable(value=[30., 40., 50., 60.], name='longitude', dimensions='lon') lat = Variable(value=[-10., -20., -30., -40., -50.], name='latitude', dimensions='lat') tas_shape = [t.shape[0], lat.shape[0], lon.shape[0]] tas = Variable(value=np.arange(reduce_multiply(tas_shape)).reshape(*tas_shape), dimensions=(dtime, 'lat', 'lon'), name='tas') time_related = Variable(value=[7, 8, 9, 10], name='time_related', dimensions=dtime) garbage1 = Variable(value=[66, 67, 68], dimensions='three', name='garbage1') dmap = {'time': {'variable': t.name}, 'x': {'variable': lon.name, DimensionMapKey.DIMENSION: [lon.dimensions[0].name]}, 'y': {'variable': lat.name, DimensionMapKey.DIMENSION: [lat.dimensions[0].name]}} field = Field(variables=[t, lon, lat, tas, garbage1, time_related], dimension_map=dmap, is_data=tas.name) return field
def get_field(self, ntime=2, variable_name='foo', nrow=2, ncol=2): """Create random field where mean varies with radius and std with the angle around the center of the grid. """ np.random.seed(1) row = Variable(value=np.arange(nrow) - nrow / 2., name='row', dimensions='row') col = Variable(value=np.arange(ncol) - ncol / 2., name='col', dimensions='col') grid = Grid(col, row) x, y = grid.get_value_stacked() start = dt.datetime(2000, 1, 1) delta = dt.timedelta(days=1) value_temporal = [start + i * delta for i in range(ntime)] temporal = TemporalVariable(value=value_temporal, dimensions='time', name='time') nlevel = 1 level = None nrlz = 1 realization = None value = np.random.rand(nrlz, ntime, nlevel, nrow, ncol) * np.arctan2(x, y).clip(.1) + np.hypot(x, y) variable = Variable(name=variable_name, value=value, dimensions=['realization', 'time', 'level', 'row', 'col']) field = Field(grid=grid, time=temporal, is_data=variable, level=level, realization=realization) return field
def test_system_get_field_dimensioned_variables(self): """Test data is appropriately tagged to identify dimensioned variables.""" path = self.get_temporary_file_path('foo.nc') time = TemporalVariable(value=[1, 2, 3], dimensions='time') x = Variable(name='x', value=[10, 20], dimensions='x') y = Variable(name='y', value=[30, 40, 50, 60], dimensions='y') data1 = Variable(name='data1', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data2 = Variable(name='data2', value=np.random.rand(3, 4, 2), dimensions=['time', 'y', 'x']) data3 = Variable(name='data3', value=[11, 12, 13], dimensions=['time']) field = Field(time=time, grid=Grid(x, y), variables=[data1, data2, data3]) field.write(path) # Test dimensioned variables are read from a file with appropriate metadata. rd = RequestDataset(path) self.assertEqual(rd.variable, ('data1', 'data2')) read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data1', 'data2')) # Test dimensioned variables are overloaded. rd = RequestDataset(path, variable='data2') read_field = rd.get() actual = get_variable_names(read_field.data_variables) self.assertEqual(actual, ('data2', ))
def get_wrap_field(crs=None, unwrapped=True): ompi = OcgDist() ompi.create_dimension('x', 5, dist=False) ompi.create_dimension('y', 7, dist=True) ompi.create_dimension('time', size_current=4, dist=False) ompi.update_dimension_bounds() if MPI_RANK == 0: row = Variable(value=[-60, -40, -20, 0, 20, 40, 60], name='y', dimensions='y') if unwrapped: col_value = [1, 90, 180, 225, 270] else: col_value = [-170, -85, 0, 85, 170] col = Variable(value=col_value, name='x', dimensions='x') grid = Grid(col, row) value = np.zeros((4, 7, 5)) for col_idx in range(value.shape[-1]): value[:, :, col_idx] = col_idx time = TemporalVariable(name='time', value=[1, 2, 3, 4], dimensions='time') var = Variable(name='foo', value=value, dimensions=['time', 'y', 'x']) field = Field(grid=grid, is_data=var, crs=crs, time=time) else: field = None field = variable_collection_scatter(field, ompi) return field
def add_data_variable_to_grid(grid): ydim, xdim = grid.dimensions tdim = Dimension(name='time', size=None) value = np.random.rand(31, ydim.size, xdim.size) data = Variable(name='data', dimensions=[tdim, ydim, xdim], value=value) tvar = TemporalVariable(name='time', value=list(range(31)), dimensions=tdim, attrs={'axis': 'T'}) grid.parent.add_variable(data) grid.parent.add_variable(tvar)
def test_system_dimension_map_formatting(self): """Test any formatting of the incoming dimension map by the field.""" dmap = {'time': {'variable': 'time'}} time = TemporalVariable(name='time', value=[1, 2, 3], dimensions='the_time') field = Field(time=time, dimension_map=dmap) actual = field.dimension_map.get_dimension('time') self.assertEqual(actual, ['the_time'])
def test_system_parallel_write_ndvariable(self): """Test a parallel CSV write with a n-dimensional variable.""" ompi = OcgDist() ompi.create_dimension('time', 3) ompi.create_dimension('extra', 2) ompi.create_dimension('x', 4) ompi.create_dimension('y', 7, dist=True) ompi.update_dimension_bounds() if MPI_RANK == 0: path = self.get_temporary_file_path('foo.csv') t = TemporalVariable(name='time', value=[1, 2, 3], dtype=float, dimensions='time') t.set_extrapolated_bounds('the_time_bounds', 'bounds') extra = Variable(name='extra', value=[7, 8], dimensions='extra') x = Variable(name='x', value=[9, 10, 11, 12], dimensions='x', dtype=float) x.set_extrapolated_bounds('x_bounds', 'bounds') # This will have the distributed dimension. y = Variable(name='y', value=[13, 14, 15, 16, 17, 18, 19], dimensions='y', dtype=float) y.set_extrapolated_bounds('y_bounds', 'bounds') data = Variable(name='data', value=np.random.rand(3, 2, 7, 4), dimensions=['time', 'extra', 'y', 'x']) vc = VariableCollection(variables=[t, extra, x, y, data]) else: path, vc = [None] * 2 path = MPI_COMM.bcast(path) vc = variable_collection_scatter(vc, ompi) with vm.scoped_by_emptyable('write', vc): if not vm.is_null: vc.write(path, iter_kwargs={'variable': 'data', 'followers': ['time', 'extra', 'y', 'x']}, driver=DriverCSV) if MPI_RANK == 0: desired = 169 with open(path, 'r') as f: lines = f.readlines() self.assertEqual(len(lines), desired)
def test_iter_masking_and_driver(self): """Test mask is set to None.""" time = TemporalVariable(value=[3, 4, 5], dimensions='time') data = Variable(value=[7, 8, 9], name='data', mask=[False, True, False], dimensions='time') field = Field(time=time, is_data=data, variables=data) itr = field.iter(allow_masked=True) actual = list(itr) self.assertIsNone(actual[1][1][data.name])
def time(self): """ :return: Get the field's time variable. Return ``None`` if no time is assigned. :rtype: :class:`~ocgis.TemporalVariable` | ``None`` """ ret = get_field_property(self, 'time') if ret is not None: if not isinstance(ret, TemporalGroupVariable): ret = TemporalVariable.from_variable(ret, format_time=self.format_time) return ret
def test_system_properties(self): """Test field properties.""" time = TemporalVariable(value=[20, 30, 40], dimensions=['the_time'], dtype=float, name='time') time_bounds = TemporalVariable(value=[[15, 25], [25, 35], [35, 45]], dimensions=['times', 'bounds'], dtype=float, name='time_bounds') other = Variable(value=[44, 55, 66], name='other', dimensions=['times_again']) x = Variable(value=[1, 2, 3], name='xc', dimensions=['x']) y = Variable(value=[10, 20, 30, 40], name='yc', dimensions=['y']) crs = CoordinateReferenceSystem(epsg=2136) f = self.get_ocgfield(variables=[time, time_bounds, other, x, y]) f2 = deepcopy(f) self.assertIsNone(f.realization) self.assertIsNone(f.time) f.dimension_map.set_variable('time', time.name) self.assertNumpyAll(f.time.get_value(), time.get_value()) self.assertEqual(f.time.attrs['axis'], 'T') self.assertIsNone(f.time.bounds) f.dimension_map.set_variable('time', 'time', bounds=time_bounds.name) self.assertNumpyAll(f.time.bounds.get_value(), time_bounds.get_value()) self.assertIn('other', f.time.parent) dims = f.dimension_map.get_dimension('time') dims += ['times', 'times_again', 'the_time'] sub = f.get_field_slice({'time': slice(1, 2)}) desired = OrderedDict([('time', (1,)), ('time_bounds', (1, 2)), ('other', (1,)), ('xc', (3,)), ('yc', (4,))]) self.assertEqual(sub.shapes, desired) self.assertIsNone(sub.grid) sub.dimension_map.set_variable('x', 'xc') sub.dimension_map.set_variable('y', 'yc') # Test writing to netCDF will load attributes. path = self.get_temporary_file_path('foo.nc') sub.write(path) with self.nc_scope(path) as ds: self.assertEqual(ds.variables[x.name].axis, 'X') self.assertEqual(ds.variables[y.name].axis, 'Y') self.assertEqual(sub.x.attrs['axis'], 'X') self.assertEqual(sub.y.attrs['axis'], 'Y') self.assertIsInstance(sub.grid, Grid) desired = OrderedDict([('time', (1,)), ('time_bounds', (1, 2)), ('other', (1,)), ('xc', (3,)), ('yc', (4,))]) self.assertEqual(sub.shapes, desired) # Test a subset. bbox = [1.5, 15, 2.5, 35] data = Variable(name='data', value=np.random.rand(3, 4), dimensions=['x', 'y']) f2.add_variable(data) f2.dimension_map.set_variable('x', 'xc') f2.dimension_map.set_variable('y', 'yc') bbox = box(*bbox) spatial_sub = f2.grid.get_intersects(bbox).parent desired = OrderedDict([('time', (3,)), ('time_bounds', (3, 2)), ('other', (3,)), ('xc', (1,)), ('yc', (2,)), ('data', (1, 2)), ]) self.assertEqual(spatial_sub.shapes, desired)
def init_variable_value(self, variable): """Set the variable value from source data conforming units in the process.""" from ocgis.variable.temporal import TemporalVariable value = self.get_variable_value(variable) variable.set_value(value, update_mask=True) # Conform the units if requested. Need to check if this variable is inside a group to find the appropriate # metadata. meta = get_variable_metadata_from_request_dataset(self, variable) conform_units_to = meta.get('conform_units_to') if conform_units_to is not None: # The initialized units for the variable are overloaded by the destination / conform to units. if isinstance(variable, TemporalVariable): from_units = TemporalVariable(units=meta['attrs']['units'], calendar=meta['attrs'].get('calendar')) from_units = from_units.cfunits else: from_units = meta['attrs']['units'] variable.cfunits_conform(conform_units_to, from_units=from_units)
def test_system_request_dataset_modifiers(self): """ Test request dataset arguments are applied in operations to fields. There are parameters that may be passed to a request dataset or used solely in operations. """ def _the_func_(arr, bounds=None): tfret = [] for ctr, element in enumerate(arr.flat): if element.year == 2004: tfret.append(ctr) return tfret time = TemporalVariable(name='my_time', value=[400, 800, 1200, 1600, 2000, 2400], dimensions='time_dimension', units='days since 2001-1-1') level = Variable(name='my_level', value=[20, 30, 40, 50], dimensions='level_dimension') np.random.seed(1) original_value = np.random.rand(time.shape[0], level.shape[0]) data = Variable(name='data', value=original_value, dimensions=['time_dimension', 'level_dimension'], units='fahrenheit') field = Field(time=time, level=level, is_data=data) ops = OcgOperations(dataset=field, time_range=[datetime.datetime(2003, 1, 1), datetime.datetime(2007, 1, 1)], time_region={'year': [2003, 2004, 2005]}, time_subset_func=_the_func_, level_range=[30, 40], conform_units_to='celsius') ret = ops.execute() actual = ret.get_element() self.assertEqual(actual.time.shape, (1,)) self.assertEqual(actual.time.value_datetime[0].year, 2004) self.assertEqual(actual.level.get_value().tolist(), [30, 40]) self.assertAlmostEqual(actual.data_variables[0].get_value().mean(), -17.511663542109229) self.assertEqual(actual.data_variables[0].units, 'celsius')
def test_system_through_operations(self): """Test calculation through operations.""" row = Variable(name='y', value=[1, 2, 3, 4], dimensions='y') col = Variable(name='x', value=[10, 11, 12], dimensions='x') grid = Grid(col, row) time = TemporalVariable(name='time', value=[1, 2], dimensions='time') data = Variable(name='data', dimensions=[time.dimensions[0]] + list(grid.dimensions)) data.get_value()[0, :] = 1 data.get_value()[1, :] = 2 field = Field(grid=grid, time=time, is_data=data) calc = [{'func': 'sum', 'name': 'sum'}] ops = OcgOperations(dataset=field, calc=calc, calc_grouping='day', calc_raw=True, aggregate=True) ret = ops.execute() actual = ret.get_element( variable_name='sum').get_masked_value().flatten() self.assertNumpyAll(actual, np.ma.array([12.0, 24.0]))