def test_computational_nc_output(self): rd = self.test_data.get_rd('cancm4_tasmax_2011', kwds={ 'time_range': [ datetime.datetime(2011, 1, 1), datetime.datetime(2011, 12, 31) ] }) calc = [{'func': 'mean', 'name': 'tasmax_mean'}] calc_grouping = ['month', 'year'] ops = ocgis.OcgOperations(rd, calc=calc, calc_grouping=calc_grouping, output_format='nc') ret = ops.execute() ds = nc.Dataset(ret, 'r') ref = ds.variables['time'] self.assertEqual(ref.climatology, 'climatology_bnds') self.assertEqual(len(ref[:]), 12) ref = ds.variables['climatology_bnds'] self.assertEqual(ref[:].shape[0], 12) ds.close() ops = ocgis.OcgOperations(dataset={ 'uri': ret, 'variable': calc[0]['name'] }, output_format='nc', prefix='subset_climatology') ret = ops.execute() # subprocess.check_call(['ncdump','-h',ret]) ip = ocgis.Inspect(ret, variable='n') ds = nc.Dataset(ret, 'r') ref = ds.variables['time'][:] self.assertEqual(len(ref), 12) self.assertEqual( set(ds.variables['tasmax_mean'].ncattrs()), set([u'_FillValue', u'units', u'long_name', u'standard_name'])) ds.close()
def test_computational_nc_output(self): rd = self.test_data.get_rd('cancm4_tasmax_2011', kwds={ 'time_range': [ datetime.datetime(2011, 1, 1), datetime.datetime(2011, 12, 31) ] }) calc = [{'func': 'mean', 'name': 'tasmax_mean'}] calc_grouping = ['month', 'year'] ops = ocgis.OcgOperations(rd, calc=calc, calc_grouping=calc_grouping) ret = ops.execute() ops = ocgis.OcgOperations(rd, calc=calc, calc_grouping=calc_grouping, output_format='nc') ret = ops.execute() ip = ocgis.Inspect(ret, variable='n') ds = nc.Dataset(ret, 'r') ref = ds.variables['time'][:] self.assertEqual(len(ref), 12) ds.close()
def write_projection_file(ds, pidx): print('writing projection: {0}'.format(pidx + 1)) ## path to the nc file holding a single projection out_path = os.path.join(OUTDIR, OUT_TEMPLATE.format(str(pidx + 1).zfill(2))) ## open the output dataset with the same format as the input ds_out = nc.Dataset(out_path, 'w', format=ds.file_format) try: ## transfer attributes to new datasets ds_out.setncatts(ds.__dict__) ## construct output dimensions - note the projection dimension is missing for dimname in ['time', 'latitude', 'longitude']: ds_out.createDimension(dimname, size=len(ds.dimensions[dimname])) ## create the coordinate variables for varname in ['time', 'latitude', 'longitude']: r_dvar = ds.variables[varname] var = ds_out.createVariable(varname, r_dvar.dtype, r_dvar.dimensions) ## remove bounds attribute as this does not exist in the dataset new_attrs = r_dvar.__dict__.copy() new_attrs.pop('bounds', None) var.setncatts(new_attrs) ## this is where the data is pulled and assigned to the new variable. ## the sync at the end is not really necessary for smaller data blocks ## but may be useful for larger files var[:] = r_dvar[:] ds_out.sync() ## create and fill the data variable using the projection index orig_var = ds.variables['Tavg'] data_var = ds_out.createVariable('Tavg', orig_var.dtype, ('time', 'latitude', 'longitude')) data_var.setncatts(orig_var.__dict__) data_var[:] = orig_var[pidx, :, :, :] finally: ds_out.close() ## ensure data may be read by OCGIS ocgis.Inspect(out_path, variable='Tavg')
def test_compute(self): # ocgis.env.VERBOSE = True # ocgis.env.DEBUG = True verbose = False n_tile_dimensions = 1 tile_range = [100, 100] rd = RequestDatasetCollection( self.test_data.get_rd('cancm4_tasmax_2011')) calc = [{ 'func': 'mean', 'name': 'my_mean' }, { 'func': 'freq_perc', 'name': 'perc_90', 'kwds': { 'percentile': 90, } }, { 'func': 'freq_perc', 'name': 'perc_95', 'kwds': { 'percentile': 95, } }, { 'func': 'freq_perc', 'name': 'perc_99', 'kwds': { 'percentile': 99, } }] calc_grouping = ['month'] ## perform computations the standard way if verbose: print('computing standard file...') ops = ocgis.OcgOperations(dataset=rd, output_format='nc', calc=calc, calc_grouping=calc_grouping, prefix='std') std_file = ops.execute() if verbose: print('standard file is: {0}'.format(std_file)) std_ds = nc.Dataset(std_file, 'r') std_meta = ocgis.Inspect(std_file).meta for ii in range(n_tile_dimensions): tile_dimension = np.random.random_integers(tile_range[0], tile_range[1]) if verbose: print('tile dimension: {0}'.format(tile_dimension)) ## perform computations using tiling tile_file = compute(rd, calc, calc_grouping, tile_dimension, verbose=verbose, prefix='tile') ## ensure output paths are different self.assertNotEqual(tile_file, std_file) ## confirm each variable is identical tile_ds = nc.Dataset(tile_file, 'r') ## compare calculated values for element in calc: tile_value, std_value = [ ds.variables[element['name']][:] for ds in [tile_ds, std_ds] ] cmp = tile_value == std_value self.assertTrue(cmp.all()) ## compare meta dictionaries tile_meta = ocgis.Inspect(tile_file).meta for k in tile_meta.iterkeys(): for k2, v2 in tile_meta[k].iteritems(): ref = std_meta[k][k2] self.assertEqual(v2, ref) tile_ds.close() std_ds.close()