def compute(dataset, calc, calc_grouping, tile_dimension, verbose=False, prefix=None): tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise (ValueError('"tile_dimension" must be greater than 0')) orig_oc = ocgis.env.OPTIMIZE_FOR_CALC ocgis.env.OPTIMIZE_FOR_CALC = True try: ods = NcDataset(request_dataset=dataset) shp = ods.spatial.grid.shape if verbose: print('getting schema...') schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension) if verbose: print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=dataset, file_only=True, calc=calc, calc_grouping=calc_grouping, output_format='nc', prefix=prefix).execute() if verbose: print('output file is: {0}'.format(fill_file)) if verbose: lschema = len(schema) print('tile count: {0}'.format(lschema)) fds = nc.Dataset(fill_file, 'a') if verbose: progress = ProgressBar('tiles progress') for ctr, indices in enumerate(schema.itervalues(), start=1): row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=dataset, slice=[None, row, col], calc=calc, calc_grouping=calc_grouping).execute() for variable in ret[1].variables.iterkeys(): ref = ret[1].calc[variable] for k, v in ref.iteritems(): vref = fds.variables[k] if len(vref.shape) == 3: vref[:, row[0]:row[1], col[0]:col[1]] = v elif len(vref.shape) == 4: vref[:, :, row[0]:row[1], col[0]:col[1]] = v else: raise (NotImplementedError) fds.sync() if verbose: progress.progress(int((float(ctr) / lschema) * 100)) fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc if verbose: progress.endProgress() print('complete.') return (fill_file)
def compute(dataset,calc,calc_grouping,tile_dimension,verbose=False,prefix=None): tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise(ValueError('"tile_dimension" must be greater than 0')) orig_oc = ocgis.env.OPTIMIZE_FOR_CALC ocgis.env.OPTIMIZE_FOR_CALC = True try: ods = NcDataset(request_dataset=dataset) shp = ods.spatial.grid.shape if verbose: print('getting schema...') schema = tile.get_tile_schema(shp[0],shp[1],tile_dimension) if verbose: print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=dataset,file_only=True, calc=calc,calc_grouping=calc_grouping, output_format='nc',prefix=prefix).execute() if verbose: print('output file is: {0}'.format(fill_file)) if verbose: lschema = len(schema) print('tile count: {0}'.format(lschema)) fds = nc.Dataset(fill_file,'a') if verbose: progress = ProgressBar('tiles progress') for ctr,indices in enumerate(schema.itervalues(),start=1): row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=dataset,slice=[None,row,col], calc=calc,calc_grouping=calc_grouping).execute() for variable in ret[1].variables.iterkeys(): ref = ret[1].calc[variable] for k,v in ref.iteritems(): vref = fds.variables[k] if len(vref.shape) == 3: vref[:,row[0]:row[1],col[0]:col[1]] = v elif len(vref.shape) == 4: vref[:,:,row[0]:row[1],col[0]:col[1]] = v else: raise(NotImplementedError) fds.sync() if verbose: progress.progress(int((float(ctr)/lschema)*100)) fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc if verbose: progress.endProgress() print('complete.') return(fill_file)
def main(): ocgis.env.OPTIMIZE_FOR_CALC = True # 215 minutes uri = '/tmp/gridded_obs.tasmax.OBS_125deg.daily.1950-1999.nc' # uri = '/usr/local/climate_data/maurer/bcca/obs/tasmax/1_8deg/gridded_obs.tasmax.OBS_125deg.daily.1950.nc' # uri = '/usr/local/climate_data/daymet/tmax.nc' variable = 'tasmax' # variable = 'tmax' rd = ocgis.RequestDataset(uri,variable) import netCDF4 as nc ods = ocgis.api.dataset.dataset.OcgDataset(rd) shp = ods.i.spatial.shape print('getting schema...') schema = tile.get_tile_schema(shp[0],shp[1],100) calc = [{'func':'mean','name':'my_mean'}, {'func':'freq_perc','name':'perc_90','kwds':{'perc':90,}}, {'func':'freq_perc','name':'perc_95','kwds':{'perc':95,}}, {'func':'freq_perc','name':'perc_99','kwds':{'perc':99,}} ] print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=rd,file_only=True, calc=calc,calc_grouping=['month'], output_format='nc').execute() print fill_file, len(schema) fds = nc.Dataset(fill_file,'a') t1 = time.time() for tile_id,indices in schema.iteritems(): print tile_id row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=rd,slice_row=row,slice_column=col, calc=calc,calc_grouping=['month'], abstraction='point').execute() ref = ret[1].variables[variable].calc_value for k,v in ref.iteritems(): vref = fds.variables[k] if len(vref.shape) == 3: vref[:,row[0]:row[1],col[0]:col[1]] = v elif len(vref.shape) == 4: vref[:,:,row[0]:row[1],col[0]:col[1]] = v else: raise(NotImplementedError) fds.sync() fds.close() print((time.time()-t1)/60.0)
def test_tile_sum(self): ntests = 1000 for ii in range(ntests): nrow, ncol, tdim = [self.get_random_integer() for ii in range(3)] x = np.random.rand(nrow, ncol) y = np.empty((nrow, ncol), dtype=float) schema = tile.get_tile_schema(nrow, ncol, tdim) tidx = schema[0] row = tidx['row'] col = tidx['col'] self.assertTrue(np.all(x[row[0]:row[1], col[0]:col[1]] == x[0:tdim, 0:tdim])) running_sum = 0.0 for value in schema.itervalues(): row, col = value['row'], value['col'] slice = x[row[0]:row[1], col[0]:col[1]] y[row[0]:row[1], col[0]:col[1]] = slice running_sum += slice.sum() self.assertAlmostEqual(running_sum, x.sum()) self.assertTrue(np.all(x == y))
def test_tile_sum(self): ntests = 1000 for ii in range(ntests): nrow, ncol, tdim = [self.get_random_integer() for ii in range(3)] x = np.random.rand(nrow, ncol) y = np.empty((nrow, ncol), dtype=float) schema = tile.get_tile_schema(nrow, ncol, tdim) tidx = schema[0] row = tidx['row'] col = tidx['col'] self.assertTrue(np.all(x[row[0]:row[1], col[0]:col[1]] == x[0:tdim, 0:tdim])) running_sum = 0.0 for value in schema.values(): row, col = value['row'], value['col'] slice = x[row[0]:row[1], col[0]:col[1]] y[row[0]:row[1], col[0]:col[1]] = slice running_sum += slice.sum() self.assertAlmostEqual(running_sum, x.sum()) self.assertTrue(np.all(x == y))
def compute(ops, tile_dimension, verbose=False, use_optimizations=True): """ Used for computations on large arrays where memory limitations are a consideration. It is is also useful for extracting data from a server that has limitations on the size of requested data arrays. This function creates an empty destination NetCDF file that is then filled by executing the operations on chunks of the requested target dataset(s) and filling the destination NetCDF file. :param ops: The target operations to tile. There must be a calculation associated with the operations. :type ops: :class:`ocgis.OcgOperations` :param int tile_dimension: The target tile/chunk dimension. This integer value must be greater than zero. :param bool verbose: If ``True``, print more verbose information to terminal. :param bool use_optimizations: If ``True``, cache :class:`Field` and :class:`TemporalGroupDimension` objects for reuse during tile iteration. :raises: AssertionError, ValuError :returns: Path to the output NetCDF file. :rtype: str >>> from ocgis import RequestDataset, OcgOperations >>> from ocgis.util.large_array import compute >>> rd = RequestDataset(uri='/path/to/file',variable='tas') >>> ops = OcgOperations(dataset=rd,calc=[{'func':'mean','name':'mean'}],output_format='nc') >>> ret = compute(ops, 25) """ # validate arguments assert isinstance(ops, OcgOperations) assert ops.calc is not None assert ops.output_format == "nc" # ensure that progress is not showing 100% at first if ops.callback is not None: orgcallback = ops.callback def zeropercentagecallback(p, m): orgcallback(0.0, m) ops.callback = zeropercentagecallback tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise (ValueError('"tile_dimension" must be greater than 0')) # determine if we are working with a multivariate function if OcgCalculationEngine._check_calculation_members_(ops.calc, AbstractMultivariateFunction): # only one multivariate calculation allowed assert len(ops.calc) == 1 has_multivariate = True else: # only one calculation allowed assert len(ops.dataset) == 1 has_multivariate = False # work on a copy of the operations to create the template file ops_file_only = deepcopy(ops) # we need the output to be file only for the first request ops_file_only.file_only = True # save the environment flag for calculation optimizations. orig_oc = ocgis.env.OPTIMIZE_FOR_CALC try: # tell the software we are optimizing for calculations ocgis.env.OPTIMIZE_FOR_CALC = True # first, write the template file if verbose: print("getting fill file...") fill_file = ops_file_only.execute() # if there is a geometry, we have to find the offset for the slice. we # also need to account for the subset mask. if ops.geom is not None: if verbose: print("geometry subset is present. calculating slice offsets...") ops_offset = deepcopy(ops) ops_offset.output_format = "numpy" ops_offset.calc = None ops_offset.agg_selection = True ops_offset.snippet = False coll = ops_offset.execute() for row in coll.get_iter_melted(): # assert the values are not loaded... assert row["variable"]._value is None # assert only 3 or 4 dimensional data is being used assert row["field"].shape_as_dict["R"] == 1 ref_spatial = coll[1][ops_offset.dataset.first().name].spatial try: row_offset = ref_spatial.grid.row._src_idx[0] col_offset = ref_spatial.grid.col._src_idx[0] except (AttributeError, TypeError): # Likely no row and column for a 2-dimensional grid. row_offset = ref_spatial.grid._src_idx["row"][0] col_offset = ref_spatial.grid._src_idx["col"][0] mask_spatial = ref_spatial.get_mask() # otherwise the offset is zero... else: row_offset = 0 col_offset = 0 mask_spatial = None # get the shape for the tile schema if verbose: print("getting tile schema shape inputs...") # if has_multivariate == False: # shp_variable = '{0}_{1}'.format(ops.calc[0]['name'],ops.dataset[0].alias) # else: # shp_variable = ops.calc[0]['name'] shp_variable = ops.calc[0]["name"] template_rd = ocgis.RequestDataset(uri=fill_file, variable=shp_variable) template_field = template_rd.get() shp = template_field.shape[-2:] if use_optimizations: # if there is a calculation grouping, optimize for it. otherwise, pass # this value as None. try: tgd_field = ops.dataset.first().get() template_tgd = tgd_field.temporal.get_grouping(deepcopy(ops.calc_grouping)) if not has_multivariate: key = ops.dataset.first().name else: key = "_".join([__.name for __ in ops.dataset.itervalues()]) optimizations = {"tgds": {key: template_tgd}} except TypeError: optimizations = None # load the fields and pass those for optimization field_optimizations = {} for rd in ops.dataset.itervalues(): gotten_field = rd.get(format_time=ops.format_time) field_optimizations.update({rd.name: gotten_field}) optimizations = optimizations or {} optimizations["fields"] = field_optimizations else: optimizations = None if verbose: print("getting tile schema...") schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension) lschema = len(schema) # Create new callbackfunction where the 0-100% range is converted to a subset corresponding to the no. of blocks to be calculated if ops.callback is not None: percentageDone = 0 callback = ops.callback def newcallback(p, m): p = (p / lschema) + percentageDone orgcallback(p, m) ops.callback = newcallback if verbose: print("output file is: {0}".format(fill_file)) print("tile count: {0}".format(lschema)) fds = nc.Dataset(fill_file, "a") try: if verbose: progress = ProgressBar("tiles progress") if ops.callback is not None and callback: callback(0, "Initializing calculation") for ctr, indices in enumerate(schema.itervalues(), start=1): # appropriate adjust the slices to account for the spatial subset row = [ii + row_offset for ii in indices["row"]] col = [ii + col_offset for ii in indices["col"]] # copy the operations and modify arguments ops_slice = deepcopy(ops) ops_slice.geom = None ops_slice.slice = [None, None, None, row, col] ops_slice.output_format = "numpy" ops_slice.optimizations = optimizations # return the object slice ret = ops_slice.execute() for field_map in ret.itervalues(): for field in field_map.itervalues(): field_shape = field.shape_as_dict for alias, variable in field.variables.iteritems(): vref = fds.variables[alias] assert isinstance(variable.value, np.ma.MaskedArray) # we need to remove the offsets to adjust for the zero-based # fill file. slice_row = slice(row[0] - row_offset, row[1] - row_offset) slice_col = slice(col[0] - col_offset, col[1] - col_offset) # if there is a spatial mask, update accordingly if mask_spatial is not None: set_variable_spatial_mask(variable, mask_spatial, slice_row, slice_col) # squeeze out extra dimensions from ocgis fill_value = np.squeeze(variable.value) # fill the netCDF container variable adjusting for shape if len(vref.shape) == 3: reshape = (field_shape["T"], field_shape["Y"], field_shape["X"]) vref[:, slice_row, slice_col] = fill_value.reshape(*reshape) elif len(vref.shape) == 4: reshape = (field_shape["T"], field_shape["Z"], field_shape["Y"], field_shape["X"]) vref[:, :, slice_row, slice_col] = fill_value.reshape(*reshape) else: raise (NotImplementedError(vref.shape)) # write the data to disk fds.sync() if verbose: progress.progress(int((float(ctr) / lschema) * 100)) if ops.callback is not None and callback: percentageDone = (float(ctr) / lschema) * 100 finally: fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc if verbose: progress.endProgress() print("complete.") return fill_file
def test_tile_get_tile_schema(self): schema = tile.get_tile_schema(5, 5, 2) self.assertEqual(len(schema), 9) schema = tile.get_tile_schema(25, 1, 2) self.assertEqual(len(schema), 13)
def compute(ops, tile_dimension, verbose=False, use_optimizations=True): """ Used for computations on large arrays where memory limitations are a consideration. It is is also useful for extracting data from a server that has limitations on the size of requested data arrays. This function creates an empty destination NetCDF file that is then filled by executing the operations on chunks of the requested target dataset(s) and filling the destination NetCDF file. :param ops: The target operations to tile. There must be a calculation associated with the operations. :type ops: :class:`ocgis.OcgOperations` :param int tile_dimension: The target tile/chunk dimension. This integer value must be greater than zero. :param bool verbose: If ``True``, print more verbose information to terminal. :param bool use_optimizations: If ``True``, cache :class:`Field` and :class:`TemporalGroupDimension` objects for reuse during tile iteration. :raises: AssertionError, ValuError :returns: Path to the output NetCDF file. :rtype: str >>> from ocgis import RequestDataset, OcgOperations >>> from ocgis.util.large_array import compute >>> rd = RequestDataset(uri='/path/to/file', variable='tas') >>> ops = OcgOperations(dataset=rd,calc=[{'func':'mean','name':'mean'}],output_format='nc') >>> ret = compute(ops, 25) """ assert isinstance(ops, OcgOperations) assert ops.calc is not None assert ops.output_format == constants.OutputFormatName.NETCDF # Ensure that progress is not showing 100% at first. if ops.callback is not None: orgcallback = ops.callback def zeropercentagecallback(p, m): orgcallback(0., m) ops.callback = zeropercentagecallback tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise ValueError('"tile_dimension" must be greater than 0') # Determine if we are working with a multivariate function. if CalculationEngine._check_calculation_members_( ops.calc, AbstractMultivariateFunction): # Only one multivariate calculation allowed. assert len(ops.calc) == 1 has_multivariate = True else: # Only one dataset allowed. assert len(list(ops.dataset)) == 1 has_multivariate = False # work on a copy of the operations to create the template file ops_file_only = deepcopy(ops) # we need the output to be file only for the first request ops_file_only.file_only = True # save the environment flag for calculation optimizations. orig_oc = ocgis.env.OPTIMIZE_FOR_CALC try: # tell the software we are optimizing for calculations ocgis.env.OPTIMIZE_FOR_CALC = True # first, write the template file if verbose: print('getting fill file...') fill_file = ops_file_only.execute() # if there is a geometry, we have to find the offset for the slice. we # also need to account for the subset mask. if ops.geom is not None: if verbose: print( 'geometry subset is present. calculating slice offsets...') ops_offset = deepcopy(ops) ops_offset.output_format = constants.OutputFormatName.OCGIS ops_offset.calc = None ops_offset.agg_selection = True ops_offset.snippet = False coll = ops_offset.execute() for row in coll.iter_melted(tag=TagName.DATA_VARIABLES): assert row['variable']._value is None ref_field = coll.get_element() ref_grid = ref_field.grid row_offset = ref_grid.dimensions[0]._src_idx[0] col_offset = ref_grid.dimensions[1]._src_idx[0] mask_spatial = ref_grid.get_mask() # otherwise the offset is zero... else: row_offset = 0 col_offset = 0 mask_spatial = None # get the shape for the tile schema if verbose: print('getting tile schema shape inputs...') shp_variable = ops.calc[0]['name'] template_rd = ocgis.RequestDataset(uri=fill_file, variable=shp_variable) template_field = template_rd.get() shp = template_field.grid.shape if use_optimizations: # if there is a calculation grouping, optimize for it. otherwise, pass # this value as None. try: # tgd_field = ops.dataset.first().get() archetype_dataset = list(ops.dataset)[0] tgd_field = archetype_dataset.get() template_tgd = tgd_field.temporal.get_grouping( deepcopy(ops.calc_grouping)) if not has_multivariate: key = archetype_dataset.field_name else: key = '_'.join([__.field_name for __ in ops.dataset]) optimizations = {'tgds': {key: template_tgd}} except TypeError: optimizations = None # load the fields and pass those for optimization field_optimizations = {} for rd in ops.dataset: gotten_field = rd.get(format_time=ops.format_time) field_optimizations.update({rd.field_name: gotten_field}) optimizations = optimizations or {} optimizations['fields'] = field_optimizations else: optimizations = None if verbose: print('getting tile schema...') schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension) lschema = len(schema) # Create new callbackfunction where the 0-100% range is converted to a subset corresponding to the no. of # blocks to be calculated if ops.callback is not None: percentageDone = 0 callback = ops.callback def newcallback(p, m): p = (p / lschema) + percentageDone orgcallback(p, m) ops.callback = newcallback if verbose: print(('output file is: {0}'.format(fill_file))) print(('tile count: {0}'.format(lschema))) fds = nc.Dataset(fill_file, 'a') try: if verbose: progress = ProgressBar('tiles progress') if ops.callback is not None and callback: callback(0, "Initializing calculation") for ctr, indices in enumerate(iter(schema.values()), start=1): # appropriate adjust the slices to account for the spatial subset row = [ii + row_offset for ii in indices['row']] col = [ii + col_offset for ii in indices['col']] # copy the operations and modify arguments ops_slice = deepcopy(ops) ops_slice.geom = None ops_slice.slice = [None, None, None, row, col] ops_slice.output_format = constants.OutputFormatName.OCGIS ops_slice.optimizations = optimizations # return the object slice ret = ops_slice.execute() for field in ret.iter_fields(): for variable in field.data_variables: vref = fds.variables[variable.name] # we need to remove the offsets to adjust for the zero-based fill file. slice_row = slice(row[0] - row_offset, row[1] - row_offset) slice_col = slice(col[0] - col_offset, col[1] - col_offset) # if there is a spatial mask, update accordingly if mask_spatial is not None: set_variable_spatial_mask(variable, mask_spatial, slice_row, slice_col) fill_mask = field.grid.get_mask(create=True) fill_mask[:, :] = mask_spatial[slice_row, slice_col] fill_mask = np.ma.array(np.zeros(fill_mask.shape), mask=fill_mask) fds.variables[field.grid.mask_variable.name][ slice_row, slice_col] = fill_mask fill_value = variable.get_masked_value() # fill the netCDF container variable adjusting for shape if len(vref.shape) == 3: vref[:, slice_row, slice_col] = fill_value elif len(vref.shape) == 4: vref[:, :, slice_row, slice_col] = fill_value else: raise NotImplementedError(vref.shape) fds.sync() if verbose: progress.progress(int((float(ctr) / lschema) * 100)) if ops.callback is not None and callback: percentageDone = ((float(ctr) / lschema) * 100) finally: fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc if verbose: progress.endProgress() print('complete.') return fill_file
def compute(dataset,calc,calc_grouping,tile_dimension,verbose=False,prefix=None): ''' :type dataset: RequestDatasetCollection ''' assert(isinstance(dataset,RequestDatasetCollection)) assert(type(calc) in (list,tuple)) tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise(ValueError('"tile_dimension" must be greater than 0')) orig_oc = ocgis.env.OPTIMIZE_FOR_CALC ocgis.env.OPTIMIZE_FOR_CALC = False try: ## load some data into the optimize store print('loading into optimize store...') for rd in dataset: if verbose: print('request dataset',rd.alias) ocgis.env._optimize_store[rd.alias] = {} ocgis.env._optimize_store[rd.alias]['_value_datetime'] = rd.ds.temporal.value_datetime ocgis.env._optimize_store[rd.alias]['_bounds_datetime'] = rd.ds.temporal.bounds_datetime if calc_grouping is not None: rd.ds.temporal.set_grouping(calc_grouping) ocgis.env._optimize_store[rd.alias]['group'] = rd.ds.temporal.group rd._ds = None ## tell the software we are optimizing for calculations ocgis.env.OPTIMIZE_FOR_CALC = True ods = NcDataset(request_dataset=dataset[0]) shp = ods.spatial.grid.shape if verbose: print('getting schema...') schema = tile.get_tile_schema(shp[0],shp[1],tile_dimension) if verbose: print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=dataset,file_only=True, calc=calc,calc_grouping=calc_grouping, output_format='nc',prefix=prefix).execute() if verbose: print('output file is: {0}'.format(fill_file)) if verbose: lschema = len(schema) print('tile count: {0}'.format(lschema)) fds = nc.Dataset(fill_file,'a') if verbose: progress = ProgressBar('tiles progress') for ctr,indices in enumerate(schema.itervalues(),start=1): row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=dataset,slice=[None,row,col], calc=calc,calc_grouping=calc_grouping).execute() for vref,v in iter_variable_values(ret[1],fds): if len(vref.shape) == 3: vref[:,row[0]:row[1],col[0]:col[1]] = v elif len(vref.shape) == 4: vref[:,:,row[0]:row[1],col[0]:col[1]] = v else: raise(NotImplementedError(vref.shape)) fds.sync() if verbose: progress.progress(int((float(ctr)/lschema)*100)) fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc ocgis.env._optimize_store = {} if verbose: progress.endProgress() print('complete.') return(fill_file)
def compute(ops, tile_dimension, verbose=False, use_optimizations=True): """ Used for computations on large arrays where memory limitations are a consideration. It is is also useful for extracting data from a server that has limitations on the size of requested data arrays. This function creates an empty destination NetCDF file that is then filled by executing the operations on chunks of the requested target dataset(s) and filling the destination NetCDF file. :param ops: The target operations to tile. There must be a calculation associated with the operations. :type ops: :class:`ocgis.OcgOperations` :param int tile_dimension: The target tile/chunk dimension. This integer value must be greater than zero. :param bool verbose: If ``True``, print more verbose information to terminal. :param bool use_optimizations: If ``True``, cache :class:`~ocgis.Field` and :class:`~ocgis.TemporalGroupVariable` objects for reuse during tile iteration. :raises: AssertionError, ValueError :returns: Path to the output NetCDF file. :rtype: str >>> from ocgis import RequestDataset, OcgOperations >>> from ocgis.util.large_array import compute >>> rd = RequestDataset(uri='/path/to/file', variable='tas') >>> ops = OcgOperations(dataset=rd, calc=[{'func':'mean','name':'mean'}],output_format='nc') >>> ret = compute(ops, 25) """ assert isinstance(ops, OcgOperations) assert ops.output_format == constants.OutputFormatName.NETCDF # Ensure that progress is not showing 100% at first. if ops.callback is not None: orgcallback = ops.callback def zeropercentagecallback(p, m): orgcallback(0., m) ops.callback = zeropercentagecallback tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise ValueError('"tile_dimension" must be greater than 0') # Determine if we are working with a multivariate function. if ops.calc is not None: if CalculationEngine._check_calculation_members_(ops.calc, AbstractMultivariateFunction): # Only one multivariate calculation allowed. assert len(ops.calc) == 1 has_multivariate = True else: # Only one dataset allowed. assert len(list(ops.dataset)) == 1 has_multivariate = False else: has_multivariate = False # work on a copy of the operations to create the template file ops_file_only = deepcopy(ops) # we need the output to be file only for the first request if ops.calc is not None: ops_file_only.file_only = True # save the environment flag for calculation optimizations. orig_oc = ocgis.env.OPTIMIZE_FOR_CALC try: # tell the software we are optimizing for calculations ocgis.env.OPTIMIZE_FOR_CALC = True # first, write the template file if verbose: print('getting fill file...') fill_file = ops_file_only.execute() # if there is a geometry, we have to find the offset for the slice. we # also need to account for the subset mask. if ops.geom is not None: if verbose: print('geometry subset is present. calculating slice offsets...') ops_offset = deepcopy(ops) ops_offset.output_format = constants.OutputFormatName.OCGIS ops_offset.calc = None ops_offset.agg_selection = True ops_offset.snippet = False coll = ops_offset.execute() for row in coll.iter_melted(tag=TagName.DATA_VARIABLES): assert row['variable']._value is None ref_field = coll.get_element() ref_grid = ref_field.grid row_offset = ref_grid.dimensions[0]._src_idx[0] col_offset = ref_grid.dimensions[1]._src_idx[0] mask_spatial = ref_grid.get_mask() # otherwise the offset is zero... else: row_offset = 0 col_offset = 0 mask_spatial = None # get the shape for the tile schema if verbose: print('getting tile schema shape inputs...') if ops.calc is not None: shp_variable = ops.calc[0]['name'] else: shp_variable = None template_rd = ocgis.RequestDataset(uri=fill_file, variable=shp_variable) template_field = template_rd.get() shp = template_field.grid.shape if use_optimizations: # if there is a calculation grouping, optimize for it. otherwise, pass # this value as None. try: # tgd_field = ops.dataset.first().get() archetype_dataset = list(ops.dataset)[0] tgd_field = archetype_dataset.get() template_tgd = tgd_field.temporal.get_grouping(deepcopy(ops.calc_grouping)) if not has_multivariate: key = archetype_dataset.field_name else: key = '_'.join([__.field_name for __ in ops.dataset]) optimizations = {'tgds': {key: template_tgd}} except TypeError: optimizations = None # load the fields and pass those for optimization field_optimizations = {} for rd in ops.dataset: gotten_field = rd.get(format_time=ops.format_time) field_optimizations.update({rd.field_name: gotten_field}) optimizations = optimizations or {} optimizations['fields'] = field_optimizations else: optimizations = None if verbose: print('getting tile schema...') schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension) lschema = len(schema) # Create new callbackfunction where the 0-100% range is converted to a subset corresponding to the no. of # blocks to be calculated if ops.callback is not None: percentageDone = 0 callback = ops.callback def newcallback(p, m): p = (p / lschema) + percentageDone orgcallback(p, m) ops.callback = newcallback if verbose: print(('output file is: {0}'.format(fill_file))) print(('tile count: {0}'.format(lschema))) fds = nc.Dataset(fill_file, 'a') try: if verbose: progress = ProgressBar('tiles progress') if ops.callback is not None and callback: callback(0, "Initializing calculation") for ctr, indices in enumerate(iter(schema.values()), start=1): # appropriate adjust the slices to account for the spatial subset row = [ii + row_offset for ii in indices['row']] col = [ii + col_offset for ii in indices['col']] # copy the operations and modify arguments ops_slice = deepcopy(ops) ops_slice.geom = None ops_slice.slice = [None, None, None, row, col] ops_slice.output_format = constants.OutputFormatName.OCGIS ops_slice.optimizations = optimizations # return the object slice ret = ops_slice.execute() for field in ret.iter_fields(): for variable in field.data_variables: vref = fds.variables[variable.name] # we need to remove the offsets to adjust for the zero-based fill file. slice_row = slice(row[0] - row_offset, row[1] - row_offset) slice_col = slice(col[0] - col_offset, col[1] - col_offset) # if there is a spatial mask, update accordingly if mask_spatial is not None: set_variable_spatial_mask(variable, mask_spatial, slice_row, slice_col) fill_mask = field.grid.get_mask(create=True) fill_mask[:, :] = mask_spatial[slice_row, slice_col] fill_mask = np.ma.array(np.zeros(fill_mask.shape), mask=fill_mask) fds.variables[field.grid.mask_variable.name][slice_row, slice_col] = fill_mask fill_value = variable.get_masked_value() # fill the netCDF container variable adjusting for shape if len(vref.shape) == 3: vref[:, slice_row, slice_col] = fill_value elif len(vref.shape) == 4: vref[:, :, slice_row, slice_col] = fill_value else: raise NotImplementedError(vref.shape) fds.sync() if verbose: progress.progress(int((float(ctr) / lschema) * 100)) if ops.callback is not None and callback: percentageDone = ((float(ctr) / lschema) * 100) finally: fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc if verbose: progress.endProgress() print('complete.') return fill_file
def compute(dataset, calc, calc_grouping, tile_dimension, verbose=False, prefix=None): ''' :type dataset: RequestDatasetCollection ''' assert (isinstance(dataset, RequestDatasetCollection)) assert (type(calc) in (list, tuple)) tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise (ValueError('"tile_dimension" must be greater than 0')) orig_oc = ocgis.env.OPTIMIZE_FOR_CALC ocgis.env.OPTIMIZE_FOR_CALC = False try: ## load some data into the optimize store print('loading into optimize store...') for rd in dataset: if verbose: print('request dataset', rd.alias) ocgis.env._optimize_store[rd.alias] = {} ocgis.env._optimize_store[ rd.alias]['_value_datetime'] = rd.ds.temporal.value_datetime ocgis.env._optimize_store[ rd.alias]['_bounds_datetime'] = rd.ds.temporal.bounds_datetime if calc_grouping is not None: rd.ds.temporal.set_grouping(calc_grouping) ocgis.env._optimize_store[ rd.alias]['group'] = rd.ds.temporal.group rd._ds = None ## tell the software we are optimizing for calculations ocgis.env.OPTIMIZE_FOR_CALC = True ods = NcDataset(request_dataset=dataset[0]) shp = ods.spatial.grid.shape if verbose: print('getting schema...') schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension) if verbose: print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=dataset, file_only=True, calc=calc, calc_grouping=calc_grouping, output_format='nc', prefix=prefix).execute() if verbose: print('output file is: {0}'.format(fill_file)) if verbose: lschema = len(schema) print('tile count: {0}'.format(lschema)) fds = nc.Dataset(fill_file, 'a') if verbose: progress = ProgressBar('tiles progress') for ctr, indices in enumerate(schema.itervalues(), start=1): row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=dataset, slice=[None, row, col], calc=calc, calc_grouping=calc_grouping).execute() for vref, v in iter_variable_values(ret[1], fds): if len(vref.shape) == 3: vref[:, row[0]:row[1], col[0]:col[1]] = v elif len(vref.shape) == 4: vref[:, :, row[0]:row[1], col[0]:col[1]] = v else: raise (NotImplementedError(vref.shape)) fds.sync() if verbose: progress.progress(int((float(ctr) / lschema) * 100)) fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc ocgis.env._optimize_store = {} if verbose: progress.endProgress() print('complete.') return (fill_file)
def compute(dataset,calc,calc_grouping,tile_dimension,verbose=False,prefix=None): ''' :type dataset: RequestDatasetCollection ''' assert(isinstance(dataset,RequestDatasetCollection)) assert(type(calc) in (list,tuple)) tile_dimension = int(tile_dimension) if tile_dimension <= 0: raise(ValueError('"tile_dimension" must be greater than 0')) orig_oc = ocgis.env.OPTIMIZE_FOR_CALC ocgis.env.OPTIMIZE_FOR_CALC = False try: ## tell the software we are optimizing for calculations ocgis.env.OPTIMIZE_FOR_CALC = True ods = dataset[0].get() # ods = NcDataset(request_dataset=dataset[0]) shp = ods.shape[-2:] if verbose: print('getting schema...') schema = tile.get_tile_schema(shp[0],shp[1],tile_dimension) if verbose: print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=dataset,file_only=True, calc=calc,calc_grouping=calc_grouping, output_format='nc',prefix=prefix).execute() if verbose: print('output file is: {0}'.format(fill_file)) if verbose: lschema = len(schema) print('tile count: {0}'.format(lschema)) fds = nc.Dataset(fill_file,'a') if verbose: progress = ProgressBar('tiles progress') for ctr,indices in enumerate(schema.itervalues(),start=1): row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=dataset,slice=[None,None,None,row,col], calc=calc,calc_grouping=calc_grouping).execute() for field_map in ret.itervalues(): for field in field_map.itervalues(): for alias,variable in field.variables.iteritems(): vref = fds.variables[alias] if len(vref.shape) == 3: vref[:,row[0]:row[1],col[0]:col[1]] = np.squeeze(variable.value) elif len(vref.shape) == 4: vref[:,:,row[0]:row[1],col[0]:col[1]] = np.squeeze(variable.value) else: raise(NotImplementedError(vref.shape)) fds.sync() # import ipdb;ipdb.set_trace() # for vref,v in iter_variable_values(ret[1],fds): # if len(vref.shape) == 3: # vref[:,row[0]:row[1],col[0]:col[1]] = v # elif len(vref.shape) == 4: # vref[:,:,row[0]:row[1],col[0]:col[1]] = v # else: # raise(NotImplementedError(vref.shape)) # fds.sync() if verbose: progress.progress(int((float(ctr)/lschema)*100)) fds.close() finally: ocgis.env.OPTIMIZE_FOR_CALC = orig_oc if verbose: progress.endProgress() print('complete.') return(fill_file)
def main(): ocgis.env.OPTIMIZE_FOR_CALC = True # 215 minutes uri = '/tmp/gridded_obs.tasmax.OBS_125deg.daily.1950-1999.nc' # uri = '/usr/local/climate_data/maurer/bcca/obs/tasmax/1_8deg/gridded_obs.tasmax.OBS_125deg.daily.1950.nc' # uri = '/usr/local/climate_data/daymet/tmax.nc' variable = 'tasmax' # variable = 'tmax' rd = ocgis.RequestDataset(uri, variable) import netCDF4 as nc ods = ocgis.api.dataset.dataset.OcgDataset(rd) shp = ods.i.spatial.shape print('getting schema...') schema = tile.get_tile_schema(shp[0], shp[1], 100) calc = [{ 'func': 'mean', 'name': 'my_mean' }, { 'func': 'freq_perc', 'name': 'perc_90', 'kwds': { 'perc': 90, } }, { 'func': 'freq_perc', 'name': 'perc_95', 'kwds': { 'perc': 95, } }, { 'func': 'freq_perc', 'name': 'perc_99', 'kwds': { 'perc': 99, } }] print('getting fill file...') fill_file = ocgis.OcgOperations(dataset=rd, file_only=True, calc=calc, calc_grouping=['month'], output_format='nc').execute() print fill_file, len(schema) fds = nc.Dataset(fill_file, 'a') t1 = time.time() for tile_id, indices in schema.iteritems(): print tile_id row = indices['row'] col = indices['col'] ret = ocgis.OcgOperations(dataset=rd, slice_row=row, slice_column=col, calc=calc, calc_grouping=['month'], abstraction='point').execute() ref = ret[1].variables[variable].calc_value for k, v in ref.iteritems(): vref = fds.variables[k] if len(vref.shape) == 3: vref[:, row[0]:row[1], col[0]:col[1]] = v elif len(vref.shape) == 4: vref[:, :, row[0]:row[1], col[0]:col[1]] = v else: raise (NotImplementedError) fds.sync() fds.close() print((time.time() - t1) / 60.0)