def test_computational_nc_output(self):
rd = self.test_data.get_rd('cancm4_tasmax_2011',
kwds={
'time_range': [
datetime.datetime(2011, 1, 1),
datetime.datetime(2011, 12, 31)
]
})
calc = [{'func': 'mean', 'name': 'tasmax_mean'}]
calc_grouping = ['month', 'year']
ops = ocgis.OcgOperations(rd,
calc=calc,
calc_grouping=calc_grouping,
output_format='nc')
ret = ops.execute()
ds = nc.Dataset(ret, 'r')
ref = ds.variables['time']
self.assertEqual(ref.climatology, 'climatology_bnds')
self.assertEqual(len(ref[:]), 12)
ref = ds.variables['climatology_bnds']
self.assertEqual(ref[:].shape[0], 12)
ds.close()
ops = ocgis.OcgOperations(dataset={
'uri': ret,
'variable': calc[0]['name']
},
output_format='nc',
prefix='subset_climatology')
ret = ops.execute()
# subprocess.check_call(['ncdump','-h',ret])
ip = ocgis.Inspect(ret, variable='n')
ds = nc.Dataset(ret, 'r')
ref = ds.variables['time'][:]
self.assertEqual(len(ref), 12)
self.assertEqual(
set(ds.variables['tasmax_mean'].ncattrs()),
set([u'_FillValue', u'units', u'long_name', u'standard_name']))
ds.close()
def test_computational_nc_output(self):
rd = self.test_data.get_rd('cancm4_tasmax_2011',
kwds={
'time_range': [
datetime.datetime(2011, 1, 1),
datetime.datetime(2011, 12, 31)
]
})
calc = [{'func': 'mean', 'name': 'tasmax_mean'}]
calc_grouping = ['month', 'year']
ops = ocgis.OcgOperations(rd, calc=calc, calc_grouping=calc_grouping)
ret = ops.execute()
ops = ocgis.OcgOperations(rd,
calc=calc,
calc_grouping=calc_grouping,
output_format='nc')
ret = ops.execute()
ip = ocgis.Inspect(ret, variable='n')
ds = nc.Dataset(ret, 'r')
ref = ds.variables['time'][:]
self.assertEqual(len(ref), 12)
ds.close()
def write_projection_file(ds, pidx):
print('writing projection: {0}'.format(pidx + 1))
## path to the nc file holding a single projection
out_path = os.path.join(OUTDIR,
OUT_TEMPLATE.format(str(pidx + 1).zfill(2)))
## open the output dataset with the same format as the input
ds_out = nc.Dataset(out_path, 'w', format=ds.file_format)
try:
## transfer attributes to new datasets
ds_out.setncatts(ds.__dict__)
## construct output dimensions - note the projection dimension is missing
for dimname in ['time', 'latitude', 'longitude']:
ds_out.createDimension(dimname, size=len(ds.dimensions[dimname]))
## create the coordinate variables
for varname in ['time', 'latitude', 'longitude']:
r_dvar = ds.variables[varname]
var = ds_out.createVariable(varname, r_dvar.dtype,
r_dvar.dimensions)
## remove bounds attribute as this does not exist in the dataset
new_attrs = r_dvar.__dict__.copy()
new_attrs.pop('bounds', None)
var.setncatts(new_attrs)
## this is where the data is pulled and assigned to the new variable.
## the sync at the end is not really necessary for smaller data blocks
## but may be useful for larger files
var[:] = r_dvar[:]
ds_out.sync()
## create and fill the data variable using the projection index
orig_var = ds.variables['Tavg']
data_var = ds_out.createVariable('Tavg', orig_var.dtype,
('time', 'latitude', 'longitude'))
data_var.setncatts(orig_var.__dict__)
data_var[:] = orig_var[pidx, :, :, :]
finally:
ds_out.close()
## ensure data may be read by OCGIS
ocgis.Inspect(out_path, variable='Tavg')
def test_compute(self):
# ocgis.env.VERBOSE = True
# ocgis.env.DEBUG = True
verbose = False
n_tile_dimensions = 1
tile_range = [100, 100]
rd = RequestDatasetCollection(
self.test_data.get_rd('cancm4_tasmax_2011'))
calc = [{
'func': 'mean',
'name': 'my_mean'
}, {
'func': 'freq_perc',
'name': 'perc_90',
'kwds': {
'percentile': 90,
}
}, {
'func': 'freq_perc',
'name': 'perc_95',
'kwds': {
'percentile': 95,
}
}, {
'func': 'freq_perc',
'name': 'perc_99',
'kwds': {
'percentile': 99,
}
}]
calc_grouping = ['month']
## perform computations the standard way
if verbose: print('computing standard file...')
ops = ocgis.OcgOperations(dataset=rd,
output_format='nc',
calc=calc,
calc_grouping=calc_grouping,
prefix='std')
std_file = ops.execute()
if verbose: print('standard file is: {0}'.format(std_file))
std_ds = nc.Dataset(std_file, 'r')
std_meta = ocgis.Inspect(std_file).meta
for ii in range(n_tile_dimensions):
tile_dimension = np.random.random_integers(tile_range[0],
tile_range[1])
if verbose: print('tile dimension: {0}'.format(tile_dimension))
## perform computations using tiling
tile_file = compute(rd,
calc,
calc_grouping,
tile_dimension,
verbose=verbose,
prefix='tile')
## ensure output paths are different
self.assertNotEqual(tile_file, std_file)
## confirm each variable is identical
tile_ds = nc.Dataset(tile_file, 'r')
## compare calculated values
for element in calc:
tile_value, std_value = [
ds.variables[element['name']][:]
for ds in [tile_ds, std_ds]
]
cmp = tile_value == std_value
self.assertTrue(cmp.all())
## compare meta dictionaries
tile_meta = ocgis.Inspect(tile_file).meta
for k in tile_meta.iterkeys():
for k2, v2 in tile_meta[k].iteritems():
ref = std_meta[k][k2]
self.assertEqual(v2, ref)
tile_ds.close()
std_ds.close()