def test_write_to_rootgrp(self):
uri = self.test_data.get_uri('narccap_wrfg')
ds = nc.Dataset(uri,'r')
meta = NcMetadata(ds)
ds.close()
crs = CFLambertConformal.load_from_metadata('pr',meta)
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = crs.write_to_rootgrp(ds)
self.assertEqual(variable.grid_mapping_name, crs.grid_mapping_name)
for k, v in crs.map_parameters_values.iteritems():
variable_v = variable.__dict__[k]
try:
self.assertEqual(variable_v, v)
except ValueError:
self.assertNumpyAll(variable_v, v)
with nc_scope(path) as ds:
meta2 = NcMetadata(ds)
meta['variables']['Lambert_Conformal'] = meta2['variables']['Lambert_Conformal']
crs2 = CFLambertConformal.load_from_metadata('pr', meta)
self.assertEqual(crs, crs2)
path2 = os.path.join(self.current_dir_output, 'foo2.nc')
with nc_scope(path2, 'w') as ds:
crs2.write_to_rootgrp(ds)
def test_write_netcdf_without_row_column_on_grid(self):
"""Test writing a field without rows and columns on the grid."""
field = self.get_field(with_value=True, with_realization=False)
field.spatial.grid.value
field.spatial.grid.corners
field.spatial.grid.row = None
field.spatial.grid.col = None
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
field.write_netcdf(ds)
self.assertAsSetEqual(ds.variables.keys(), ['time', 'time_bounds', 'level', 'level_bounds',
constants.DEFAULT_NAME_ROW_COORDINATES,
constants.DEFAULT_NAME_COL_COORDINATES, 'yc_corners',
'xc_corners', 'tmax'])
self.assertAsSetEqual(ds.dimensions.keys(),
['time', 'bounds', 'level', constants.DEFAULT_NAME_ROW_COORDINATES,
constants.DEFAULT_NAME_COL_COORDINATES, constants.DEFAULT_NAME_CORNERS_DIMENSION])
# test with name on the grid
field = self.get_field(with_value=True, with_realization=False)
field.spatial.grid.value
field.spatial.grid.corners
field.spatial.grid.name_row = 'nr'
field.spatial.grid.name_col = 'nc'
field.spatial.grid.row = None
field.spatial.grid.col = None
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
field.write_netcdf(ds)
self.assertAsSetEqual(ds.variables.keys(),
['time', 'time_bounds', 'level', 'level_bounds', 'nr', 'nc', 'nr_corners',
'nc_corners', 'tmax'])
self.assertAsSetEqual(ds.dimensions.keys(),
['time', 'bounds', 'level', 'nr', 'nc', constants.DEFAULT_NAME_CORNERS_DIMENSION])
def test_write_to_netcdf_object(self):
path = os.path.join(self.current_dir_output, 'foo.nc')
a, attrs = self.get_attributes()
# Write to dataset object.
with nc_scope(path, 'w') as ds:
a.write_attributes_to_netcdf_object(ds)
with nc_scope(path, 'r') as ds:
self.assertDictEqual(ds.__dict__, a.attrs)
# Write to variable object.
with nc_scope(path, 'w') as ds:
ds.createDimension('foo')
var = ds.createVariable('foo', int, dimensions=('foo', ))
a.write_attributes_to_netcdf_object(var)
with nc_scope(path, 'r') as ds:
var = ds.variables['foo']
self.assertDictEqual(var.__dict__, a.attrs)
# Test strange unicode characters are handled.
strange = u'\xfc'
attrs = Attributes(attrs={'uni': strange, 'normal': 'attribute'})
path = self.get_temporary_file_path('foo2.nc')
with self.nc_scope(path, 'w') as ds:
attrs.write_attributes_to_netcdf_object(ds)
def test_write_to_rootgrp(self):
meta = self.archetype_minimal_metadata
crs = CFLambertConformal.load_from_metadata('pr', meta)
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = crs.write_to_rootgrp(ds)
self.assertEqual(variable.grid_mapping_name, crs.grid_mapping_name)
for k, v in crs.map_parameters_values.items():
variable_v = variable.__dict__[k]
try:
self.assertEqual(variable_v, v)
except ValueError:
# Some values come back as NumPy arrays.
self.assertEqual(variable_v.tolist(), v)
with nc_scope(path) as ds:
meta2 = {'variables': {'Lambert_Conformal': {'attrs': dict(ds.variables['Lambert_Conformal'].__dict__),
'name': 'Lambert_Conformal'}}}
meta['variables']['Lambert_Conformal'] = meta2['variables']['Lambert_Conformal']
crs2 = CFLambertConformal.load_from_metadata('pr', meta)
self.assertEqual(crs, crs2)
path2 = os.path.join(self.current_dir_output, 'foo2.nc')
with nc_scope(path2, 'w') as ds:
crs2.write_to_rootgrp(ds)
def test_write_to_rootgrp(self):
meta = self.archetype_minimal_metadata
crs = CFLambertConformal.load_from_metadata('pr', meta)
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = crs.write_to_rootgrp(ds)
self.assertEqual(variable.grid_mapping_name, crs.grid_mapping_name)
for k, v in crs.map_parameters_values.items():
variable_v = variable.__dict__[k]
try:
self.assertEqual(variable_v, v)
except ValueError:
# Some values come back as NumPy arrays.
self.assertEqual(variable_v.tolist(), v)
with nc_scope(path) as ds:
meta2 = {
'variables': {
'Lambert_Conformal': {
'attrs':
dict(ds.variables['Lambert_Conformal'].__dict__),
'name': 'Lambert_Conformal'
}
}
}
meta['variables']['Lambert_Conformal'] = meta2['variables'][
'Lambert_Conformal']
crs2 = CFLambertConformal.load_from_metadata('pr', meta)
self.assertEqual(crs, crs2)
path2 = os.path.join(self.current_dir_output, 'foo2.nc')
with nc_scope(path2, 'w') as ds:
crs2.write_to_rootgrp(ds)
def test_write_to_netcdf_object(self):
path = os.path.join(self.current_dir_output, 'foo.nc')
a, attrs = self.get_attributes()
# Write to dataset object.
with nc_scope(path, 'w') as ds:
a.write_attributes_to_netcdf_object(ds)
with nc_scope(path, 'r') as ds:
self.assertDictEqual(ds.__dict__, a.attrs)
# Write to variable object.
with nc_scope(path, 'w') as ds:
ds.createDimension('foo')
var = ds.createVariable('foo', int, dimensions=('foo',))
a.write_attributes_to_netcdf_object(var)
with nc_scope(path, 'r') as ds:
var = ds.variables['foo']
self.assertDictEqual(var.__dict__, a.attrs)
# Test strange unicode characters are handled.
strange = u'\xfc'
attrs = Attributes(attrs={'uni': strange, 'normal': 'attribute'})
path = self.get_temporary_file_path('foo2.nc')
with self.nc_scope(path, 'w') as ds:
attrs.write_attributes_to_netcdf_object(ds)
def get_multiple_variable_request_dataset_dictionary(self):
rd_orig = self.test_data.get_rd('cancm4_tas')
dest_uri = os.path.join(self.current_dir_output, os.path.split(rd_orig.uri)[1])
shutil.copy2(rd_orig.uri, dest_uri)
with nc_scope(dest_uri, 'a') as ds:
var = ds.variables['tas']
outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions)
outvar[:] = var[:] + 3
outvar.setncatts(var.__dict__)
with nc_scope(dest_uri) as ds:
self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys())))
return {'uri': dest_uri, 'variable': ['tas', 'tasmax']}
def test_selecting_single_value(self):
rd = self.test_data.get_rd('cancm4_tas')
lat_index = 32
lon_index = 97
with nc_scope(rd.uri) as ds:
lat_value = ds.variables['lat'][lat_index]
lon_value = ds.variables['lon'][lon_index]
data_values = ds.variables['tas'][:, lat_index, lon_index]
ops = ocgis.OcgOperations(dataset=rd, geom=[lon_value, lat_value])
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
geom = Point(lon_value, lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd, geom=geom)
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
geom = Point(lon_value - 360., lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd, geom=geom)
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
geom = Point(lon_value - 360., lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd,
geom=geom,
aggregate=True,
spatial_operation='clip')
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
ops = ocgis.OcgOperations(dataset=rd,
geom=[lon_value, lat_value],
search_radius_mult=0.1,
output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
values = np.squeeze(ds.variables['tas'][:])
self.assertNumpyAll(data_values, values)
def test_get_field_different_dimension_names_and_values(self):
"""Test dimension names and dimension values are correctly read from netCDF."""
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
ds.createDimension('lat', 1)
ds.createDimension('lon', 1)
ds.createDimension('tme', 1)
ds.createDimension('the_bounds', 2)
latitude = ds.createVariable('latitude', int, dimensions=('lat',))
longitude = ds.createVariable('longitude', int, dimensions=('lon',))
time = ds.createVariable('time', int, dimensions=('tme',))
time_bounds = ds.createVariable('long_live_the_bounds', int, dimensions=('tme', 'the_bounds'))
time.units = 'days since 0000-01-01'
time.bounds = 'long_live_the_bounds'
value = ds.createVariable('value', int, dimensions=('tme', 'lat', 'lon'))
latitude[:] = 5
longitude[:] = 6
time[:] = 6
value[:] = np.array([7]).reshape(1, 1, 1)
rd = RequestDataset(path)
driver = DriverNetcdf(rd)
field = driver._get_field_()
self.assertEqual(field.temporal.name, 'tme')
self.assertEqual(field.temporal.name_value, 'time')
self.assertEqual(field.spatial.grid.row.name, 'lat')
self.assertEqual(field.spatial.grid.row.name_value, 'latitude')
self.assertEqual(field.spatial.grid.col.name, 'lon')
self.assertEqual(field.spatial.grid.col.name_value, 'longitude')
self.assertEqual(field.temporal.name_bounds, 'long_live_the_bounds')
def test_calculation_operations_to_nc(self):
rd = self.test_data.get_rd('cancm4_tasmax_2011')
slc = [None, None, None, [0, 10], [0, 10]]
ops_ocgis = OcgOperations(calc=[{'func': 'icclim_SU', 'name': 'SU'}], calc_grouping=['month'], slice=slc,
dataset=rd, output_format='nc')
ret = ops_ocgis.execute()
with nc_scope(ret) as ds:
to_test = deepcopy(ds.__dict__)
history = to_test.pop('history')
self.assertEqual(history[111:187],
' Calculation of SU indice (monthly climatology) from 2011-1-1 to 2020-12-31.')
actual = OrderedDict([('source_data_global_attributes',
'{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2010", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "64384802-3f0f-4ab4-b569-697bd5430854", "branch_time_YMDH": "2011:01:01:00", "CCCma_runid": "DHFP1B_E002_I2011_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2010", "frequency": "day", "creation_date": "2012-03-28T15:32:08Z", "history": "2012-03-28T15:32:08Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2010", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.8.0"}'),
('title', 'ECA heat indice SU'), (
'references',
'ATBD of the ECA indices calculation (http://eca.knmi.nl/documents/atbd.pdf)'),
('institution', 'Climate impact portal (http://climate4impact.eu)'),
('comment', ' ')])
self.assertDictEqual(to_test, actual)
var = ds.variables['SU']
to_test = dict(var.__dict__)
to_test.pop('_FillValue', None)
self.assertEqual(to_test, {'units': 'days',
'standard_name': AbstractIcclimFunction.standard_name,
'long_name': 'Summer days (number of days where daily maximum temperature > 25 degrees)',
'grid_mapping': 'latitude_longitude'})
def test_multiple_variables(self):
conv_klasses = [CsvConverter, NcConverter]
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
var2 = deepcopy(field['tas'].extract())
var2.set_name('tas2')
field.add_variable(var2, is_data=True)
field = field.get_field_slice({'time': slice(0, 2), 'y': slice(0, 5), 'x': slice(0, 5)})
coll = self.get_spatial_collection(field=field)
for conv_klass in conv_klasses:
if conv_klass == CsvConverter:
kwds = {'melted': True}
else:
kwds = {}
prefix = 'ocgis_output_{0}'.format(conv_klass.__name__)
conv = conv_klass([coll], outdir=self.current_dir_output, prefix=prefix, **kwds)
ret = conv.write()
if conv_klass == CsvConverter:
with open(ret, 'r') as f:
reader = DictReader(f)
aliases = set([row['VARIABLE'] for row in reader])
self.assertEqual(set(['tas', 'tas2']), aliases)
else:
with nc_scope(ret) as ds:
self.assertAlmostEqual(ds.variables['tas'][:].mean(), np.float32(247.08411))
self.assertNumpyAll(ds.variables['tas'][:], ds.variables['tas2'][:])
def test_get_field_nonequivalent_units_in_source_data(self):
new_path = self.test_data.copy_file('cancm4_tas', self.current_dir_output)
# put non-equivalent units on the source data and attempto to conform
with nc_scope(new_path, 'a') as ds:
ds.variables['tas'].units = 'coulomb'
rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius')
with self.assertRaises(RequestValidationError):
rd.get()
# remove units altogether
with nc_scope(new_path, 'a') as ds:
ds.variables['tas'].delncattr('units')
rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius')
with self.assertRaises(NoUnitsError):
rd.get()
def test_multiple_variables(self):
conv_klasses = [CsvConverter, NcConverter]
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
var2 = deepcopy(field['tas'].extract())
var2.set_name('tas2')
field.add_variable(var2, is_data=True)
field = field.get_field_slice({
'time': slice(0, 2),
'y': slice(0, 5),
'x': slice(0, 5)
})
coll = self.get_spatial_collection(field=field)
for conv_klass in conv_klasses:
if conv_klass == CsvConverter:
kwds = {'melted': True}
else:
kwds = {}
prefix = 'ocgis_output_{0}'.format(conv_klass.__name__)
conv = conv_klass([coll],
outdir=self.current_dir_output,
prefix=prefix,
**kwds)
ret = conv.write()
if conv_klass == CsvConverter:
with open(ret, 'r') as f:
reader = DictReader(f)
aliases = set([row['VARIABLE'] for row in reader])
self.assertEqual(set(['tas', 'tas2']), aliases)
else:
with nc_scope(ret) as ds:
self.assertAlmostEqual(ds.variables['tas'][:].mean(),
np.float32(247.08411))
self.assertNumpyAll(ds.variables['tas'][:],
ds.variables['tas2'][:])
def test_multiple_variables(self):
conv_klasses = [CsvConverter, NcConverter]
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
var2 = deepcopy(field.variables['tas'])
var2.alias = 'tas2'
field.variables.add_variable(deepcopy(var2), assign_new_uid=True)
field = field[:, 0:2, :, 0:5, 0:5]
coll = self.get_spatial_collection(field=field)
for conv_klass in conv_klasses:
if conv_klass == CsvConverter:
kwds = {'melted': True}
else:
kwds = {}
prefix = 'ocgis_output_{0}'.format(conv_klass.__name__)
conv = conv_klass([coll], outdir=self.current_dir_output, prefix=prefix, **kwds)
ret = conv.write()
if conv_klass == CsvConverter:
with open(ret, 'r') as f:
reader = DictReader(f)
aliases = set([row['ALIAS'] for row in reader])
self.assertEqual(set(['tas', 'tas2']), aliases)
else:
with nc_scope(ret) as ds:
self.assertAlmostEqual(ds.variables['tas'][:].mean(), np.float32(247.08411))
self.assertNumpyAll(ds.variables['tas'][:], ds.variables['tas2'][:])
def test_selecting_single_value(self):
rd = self.test_data.get_rd('cancm4_tas')
lat_index = 32
lon_index = 97
with nc_scope(rd.uri) as ds:
lat_value = ds.variables['lat'][lat_index]
lon_value = ds.variables['lon'][lon_index]
data_values = ds.variables['tas'][:, lat_index, lon_index]
ops = ocgis.OcgOperations(dataset=rd, geom=[lon_value, lat_value])
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
geom = Point(lon_value, lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd, geom=geom)
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
geom = Point(lon_value - 360., lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd, geom=geom)
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
geom = Point(lon_value - 360., lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd, geom=geom, aggregate=True, spatial_operation='clip')
ret = ops.execute()
actual = ret.get_element(variable_name='tas').get_masked_value()
values = np.squeeze(actual)
self.assertNumpyAll(data_values, values.data)
self.assertFalse(np.any(values.mask))
ops = ocgis.OcgOperations(dataset=rd, geom=[lon_value, lat_value],
search_radius_mult=0.1, output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
values = np.squeeze(ds.variables['tas'][:])
self.assertNumpyAll(data_values, values)
def test_write_to_rootgrp(self):
crs = CoordinateReferenceSystem(epsg=4326, name='hello_world')
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = crs.write_to_rootgrp(ds)
self.assertIsInstance(variable, nc.Variable)
with self.assertRaises(AttributeError):
variable.proj4
def test_write_to_rootgrp(self):
crs = CoordinateReferenceSystem(epsg=4326, name='hello_world')
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = crs.write_to_rootgrp(ds)
self.assertIsInstance(variable, nc.Variable)
with self.assertRaises(AttributeError):
variable.proj4
def test_write_to_rootgrp(self):
rd = self.test_data.get_rd('narccap_rotated_pole')
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = rd.crs.write_to_rootgrp(ds)
self.assertIsInstance(variable, nc.Variable)
self.assertEqual(variable.proj4, '')
self.assertEqual(variable.proj4_transform, rd.crs._trans_proj)
def test_system_file_only_through_operations(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd, calc=[{'func': 'mean', 'name': 'mean'}], calc_grouping=['month'],
geom='state_boundaries', select_ugid=[27], file_only=True, output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
var = ds.variables['mean']
# All data should be masked since this is file only.
self.assertTrue(var[:].mask.all())
def test_to_netcdf_with_geometry(self):
rd = self.test_data.get_rd('narccap_rotated_pole')
# this bounding box covers the entire spatial domain. the software will move between rotated pole and CFWGS84
# using this operation. it can then be compared against the "null" result which just does a snippet.
geom = [-173.3, 8.8, -20.6, 79.0]
ops = OcgOperations(dataset=rd, output_format='nc', snippet=True, geom=geom)
ret = ops.execute()
ops2 = OcgOperations(dataset=rd, output_format='nc', snippet=True, prefix='hi')
ret2 = ops2.execute()
self.assertNcEqual(ret, ret2, metadata_only=True, ignore_attributes={'global': ['history']})
with nc_scope(ret) as ds:
with nc_scope(ret2) as ds2:
for var_name in ['yc', 'xc', 'tas']:
var = ds.variables[var_name][:]
var2 = ds2.variables[var_name][:]
diff = np.abs(var - var2)
self.assertTrue(diff.max() <= 1.02734374963e-06)
def test_output_datatype(self):
# ensure the output data type is the same as the input data type of the variable.
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd, calc=[{'func': 'mean', 'name': 'mean'}], calc_grouping=['month'],
geom='state_boundaries', select_ugid=[27])
ret = ops.execute()
with nc_scope(rd.uri) as ds:
var_dtype = ds.variables['tas'].dtype
self.assertEqual(ret[27]['tas'].variables['mean'].dtype, var_dtype)
def test_write_to_rootgrp(self):
rd = self.test_data.get_rd('narccap_rotated_pole')
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
variable = rd.crs.write_to_rootgrp(ds, with_proj4=True)
self.assertIsInstance(variable, nc.Variable)
self.assertEqual(variable.proj4, '')
self.assertEqual(variable.proj4_transform, rd.crs._trans_proj)
def test_get_field_dtype_on_dimensions(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
with nc_scope(rd.uri) as ds:
test_dtype_temporal = ds.variables['time'].dtype
test_dtype_value = ds.variables['tas'].dtype
self.assertEqual(field.temporal.dtype, test_dtype_temporal)
self.assertEqual(field.variables['tas'].dtype, test_dtype_value)
self.assertEqual(field.temporal.dtype, np.float64)
def test_write_to_netcdf_object(self):
path = os.path.join(self.current_dir_output, 'foo.nc')
a, attrs = self.get_attributes()
# write to dataset object
with nc_scope(path, 'w') as ds:
a.write_attributes_to_netcdf_object(ds)
with nc_scope(path, 'r') as ds:
self.assertDictEqual(ds.__dict__, a.attrs)
# write to variable object
with nc_scope(path, 'w') as ds:
ds.createDimension('foo')
var = ds.createVariable('foo', int, dimensions=('foo',))
a.write_attributes_to_netcdf_object(var)
with nc_scope(path, 'r') as ds:
var = ds.variables['foo']
self.assertDictEqual(var.__dict__, a.attrs)
def check_coords():
path = '/home/benkoziol/l/data/ocgis/ugrid-cesm-subsetting/UGRID_1km-merge-10min_HYDRO1K-merge-nomask_c130402.nc'
indices = np.array([73139, 74240, 74241, 74242])
with nc_scope(path) as ds:
var = ds.variables['landmesh_node_x']
var = ds.variables['landmesh_node_y']
print var[indices]
var = ds.variables['landmesh_face_node']
print var[0, :]
def check_coords():
path = '/home/benkoziol/l/data/ocgis/ugrid-cesm-subsetting/UGRID_1km-merge-10min_HYDRO1K-merge-nomask_c130402.nc'
indices = np.array([73139, 74240, 74241, 74242])
with nc_scope(path) as ds:
var = ds.variables['landmesh_node_x']
var = ds.variables['landmesh_node_y']
print var[indices]
var = ds.variables['landmesh_face_node']
print var[0, :]
def test_system_output_datatype(self):
"""Test the output data type is the same as the input data type of the variable."""
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd, calc=[{'func': 'mean', 'name': 'mean'}], calc_grouping=['month'],
geom='state_boundaries', select_ugid=[27])
ret = ops.execute()
with nc_scope(rd.uri) as ds:
var_dtype = ds.variables['tas'].dtype
actual = ret.get_element(variable_name='mean').dtype
self.assertEqual(actual, var_dtype)
def test_write_netcdf_bounds_dimension_exists(self):
"""Test writing with bounds when the bounds dimension has already been created."""
vd = VectorDimension(value=[3., 7.], name='one')
vd.set_extrapolated_bounds()
vd2 = VectorDimension(value=[5., 6.], name='two')
vd2.set_extrapolated_bounds()
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
vd.write_netcdf(ds)
vd2.write_netcdf(ds)
self.assertEqual(ds.variables.keys(), ['one', 'one_bounds', 'two', 'two_bounds'])
def test_get_dimensioned_variables_two_variables_in_target_dataset(self):
rd_orig = self.test_data.get_rd('cancm4_tas')
dest_uri = os.path.join(self.current_dir_output, os.path.split(rd_orig.uri)[1])
shutil.copy2(rd_orig.uri, dest_uri)
with nc_scope(dest_uri, 'a') as ds:
var = ds.variables['tas']
outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions)
outvar[:] = var[:] + 3
outvar.setncatts(var.__dict__)
rd = RequestDataset(uri=dest_uri)
self.assertEqual(rd.variable, ('tas', 'tasmax'))
self.assertEqual(rd.variable, rd.alias)
def test_write_netcdf_with_metadata(self):
"""Test writing to netCDF with a source metadata dictionary attached and data loaded from file."""
rd = self.test_data.get_rd('narccap_lambert_conformal')
field = rd.get()[:, 0:31, :, 20:30, 30:40]
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
field.write_netcdf(ds)
self.assertSetEqual(set(ds.variables.keys()), {u'time', 'time_bnds', u'yc', u'xc', u'Lambert_Conformal',
'pr'})
self.assertGreater(len(ds.__dict__), 0)
self.assertGreater(len(ds.variables['time'].__dict__), 0)
def test_operations(self):
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(uri=uri, variable='tas')
calc_grouping = ['month', 'year']
calc = [{'func': 'dynamic_kernel_percentile_threshold', 'name': 'tg10p',
'kwds': {'percentile': 10, 'width': 5, 'operation': 'lt'}}]
ops = OcgOperations(dataset=rd, calc_grouping=calc_grouping, calc=calc, output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
ref = ds.variables['tg10p'][:]
self.assertAlmostEqual(ref.mean(), 2.9778006)
def test_calendar_attribute_none(self):
"""Test that the empty string is correctly interpreted as None."""
# path to the test data file
out_nc = os.path.join(self.current_dir_output, self.fn)
# case of calendar being set to an empty string
with nc_scope(out_nc, "a") as ds:
ds.variables["time"].calendar = ""
rd = ocgis.RequestDataset(uri=out_nc, variable="foo")
# the default for the calendar overload is standard
self.assertEqual(rd.t_calendar, None)
def test_with_overloads_real_data(self):
# copy the test file as the calendar attribute will be modified
rd = self.test_data.get_rd('cancm4_tas')
filename = os.path.split(rd.uri)[1]
dest = os.path.join(self.current_dir_output, filename)
shutil.copy2(rd.uri, dest)
# modify the calendar attribute
with nc_scope(dest, 'a') as ds:
self.assertEqual(ds.variables['time'].calendar, '365_day')
ds.variables['time'].calendar = '365_days'
# assert the calendar is in fact changed on the source file
with nc_scope(dest, 'r') as ds:
self.assertEqual(ds.variables['time'].calendar, '365_days')
rd2 = RequestDataset(uri=dest, variable='tas')
field = rd2.get()
# the bad calendar will raise a value error when the datetimes are converted.
with self.assertRaises(ValueError):
field.temporal.value_datetime
# overload the calendar and confirm the datetime values are the same as the datetime values from the original
# good file
rd3 = RequestDataset(uri=dest, variable='tas', t_calendar='365_day')
field = rd3.get()
self.assertNumpyAll(field.temporal.value_datetime, rd.get().temporal.value_datetime)
# pass as a dataset collection to operations and confirm the data may be written to a flat file. dates are
# converted in the process.
time_range = (datetime.datetime(2001, 1, 1, 0, 0), datetime.datetime(2011, 1, 1, 0, 0))
dataset = [{'time_region': None,
'uri': dest,
'time_range': time_range,
'alias': u'tas',
't_units': u'days since 1850-1-1',
'variable': u'tas',
't_calendar': u'365_day'}]
rdc = RequestDatasetCollection(dataset)
ops = OcgOperations(dataset=rdc, geom='state_boundaries', select_ugid=[25],
output_format=constants.OUTPUT_FORMAT_SHAPEFILE)
ops.execute()
def test_unknown_calendar_attribute(self):
"""Test a calendar attribute with an unknown calendar attribute."""
# path to the test data file
out_nc = os.path.join(self.current_dir_output, self.fn)
# case of a calendar being set a bad value but read anyway
with nc_scope(out_nc, 'a') as ds:
ds.variables['time'].calendar = 'foo'
rd = ocgis.RequestDataset(uri=out_nc, variable='foo')
field = rd.get()
self.assertEqual(field.temporal.calendar, 'foo')
# calendar is only access when the float values are converted to datetime objects
with self.assertRaises(ValueError):
field.temporal.value_datetime
# now overload the value and ensure the field datetimes may be loaded
rd = ocgis.RequestDataset(uri=out_nc, variable='foo', t_calendar='standard')
self.assertEqual(rd.source_metadata['variables']['time']['attrs']['calendar'], 'foo')
field = rd.get()
self.assertEqual(field.temporal.calendar, 'standard')
field.temporal.value_datetime
# case of a missing calendar attribute altogether
with nc_scope(out_nc, 'a') as ds:
ds.variables['time'].delncattr('calendar')
rd = ocgis.RequestDataset(uri=out_nc, variable='foo')
self.assertEqual(rd.t_calendar, None)
self.assertIsInstance(rd.inspect_as_dct(), OrderedDict)
# write the data to a netCDF and ensure the calendar is written.
ret = ocgis.OcgOperations(dataset=rd, output_format='nc').execute()
with nc_scope(ret) as ds:
self.assertEqual(ds.variables['time'].calendar, 'standard')
self.assertEqual(ds.variables['time_bnds'].calendar, 'standard')
field = rd.get()
# the standard calendar name should be available at the dataset level
self.assertEqual(field.temporal.calendar, 'standard')
def test_unknown_calendar_attribute(self):
"""Test a calendar attribute with an unknown calendar attribute."""
# Path to the test data file.
out_nc = os.path.join(self.current_dir_output, self.fn)
# Case of a calendar being set a bad value but read anyway.
with nc_scope(out_nc, "a") as ds:
ds.variables["time"].calendar = "foo"
rd = ocgis.RequestDataset(uri=out_nc, variable="foo")
field = rd.get()
self.assertEqual(field.temporal.calendar, "foo")
# Calendar is only accessed when the float values are converted to datetime objects.
with self.assertRaises(ValueError):
field.temporal.value_datetime
# now overload the value and ensure the field datetimes may be loaded
rd = ocgis.RequestDataset(uri=out_nc, variable="foo", t_calendar="standard")
self.assertEqual(rd.source_metadata["variables"]["time"]["attrs"]["calendar"], "foo")
field = rd.get()
self.assertEqual(field.temporal.calendar, "standard")
field.temporal.value_datetime
# Case of a missing calendar attribute.
with nc_scope(out_nc, "a") as ds:
ds.variables["time"].delncattr("calendar")
rd = ocgis.RequestDataset(uri=out_nc, variable="foo")
self.assertEqual(rd.t_calendar, None)
self.assertIsInstance(rd.inspect_as_dct(), OrderedDict)
# Write the data to a netCDF and ensure the calendar is written.
ret = ocgis.OcgOperations(dataset=rd, output_format="nc").execute()
with nc_scope(ret) as ds:
self.assertEqual(ds.variables["time"].calendar, "standard")
self.assertEqual(ds.variables["time_bnds"].calendar, "standard")
field = rd.get()
# The standard calendar name should be available at the dataset level.
self.assertEqual(field.temporal.calendar, "standard")
def test_calculate_rotated_pole(self):
tasmin_fake = self.test_data.get_rd('rotated_pole_ichec', kwds={'rename_variable': 'tasmin'})
tasmax_fake = self.test_data.get_rd('rotated_pole_ichec', kwds={'rename_variable': 'tasmax'})
rds = [tasmin_fake, tasmax_fake]
for rd in rds:
rd.time_region = {'year': [1973]}
calc_ETR = [{'func': 'icclim_ETR', 'name': 'ETR', 'kwds': {'tasmin': 'tasmin', 'tasmax': 'tasmax'}}]
ops = ocgis.OcgOperations(dataset=[tasmin_fake, tasmax_fake],
calc=calc_ETR,
calc_grouping=['year', 'month'],
prefix='ETR_ocgis_icclim',
output_format='nc',
add_auxiliary_files=False)
with nc_scope(ops.execute()) as ds:
self.assertEqual(ds.variables['ETR'][:].shape, (12, 103, 106))
def test_large_array_compute_local(self):
"""Test tiling works for percentile-based indice on a local dataset."""
raise SkipTest('function is very slow with ICCLIM 4.2.5')
calc = [{'func': 'icclim_TG10p', 'name': 'itg'}]
calc_grouping = ['month']
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd, calc=calc, calc_grouping=calc_grouping, output_format='nc',
geom='state_boundaries',
select_ugid=[24])
ret = compute(ops, 5, verbose=False)
with nc_scope(ret) as ds:
self.assertAlmostEqual(ds.variables['itg'][:].sum(), 2121.0, 6)
def test_system_output_datatype(self):
"""Test the output data type is the same as the input data type of the variable."""
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,
calc=[{
'func': 'mean',
'name': 'mean'
}],
calc_grouping=['month'],
geom='state_boundaries',
select_ugid=[27])
ret = ops.execute()
with nc_scope(rd.uri) as ds:
var_dtype = ds.variables['tas'].dtype
actual = ret.get_element(variable_name='mean').dtype
self.assertEqual(actual, var_dtype)
def test_system_file_only_through_operations(self):
rd = self.test_data.get_rd('cancm4_tas')
ops = ocgis.OcgOperations(dataset=rd,
calc=[{
'func': 'mean',
'name': 'mean'
}],
calc_grouping=['month'],
geom='state_boundaries',
select_ugid=[27],
file_only=True,
output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
var = ds.variables['mean']
# All data should be masked since this is file only.
self.assertTrue(var[:].mask.all())
def test_calculation_operations_to_nc(self):
rd = self.test_data.get_rd('cancm4_tas')
slc = [None, None, None, [0, 10], [0, 10]]
ops_ocgis = OcgOperations(calc=[{'func': 'icclim_TG', 'name': 'TG'}],
calc_grouping=['month'],
slice=slc,
dataset=rd,
output_format='nc')
ret = ops_ocgis.execute()
with nc_scope(ret) as ds:
self.assertIn('Calculation of TG indice (monthly climatology)', ds.history)
self.assertEqual(ds.title, 'ECA temperature indice TG')
var = ds.variables['TG']
# check the JSON serialization
actual = '{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2000", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "fac7bd83-dd7a-425b-b4dc-b5ab2e915939", "branch_time_YMDH": "2001:01:01:00", "CCCma_runid": "DHFP1B_E002_I2001_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2000", "frequency": "day", "creation_date": "2011-05-08T01:01:51Z", "history": "2011-05-08T01:01:51Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2000", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.5.4"}'
self.assertEqual(ds.__dict__[AbstractIcclimFunction._global_attribute_source_name], actual)
# load the original source attributes from the JSON string
json.loads(ds.__dict__[AbstractIcclimFunction._global_attribute_source_name])
actual = {'units': 'K', 'grid_mapping': 'latitude_longitude',
'standard_name': AbstractIcclimFunction.standard_name,
'long_name': 'Mean of daily mean temperature'}
self.assertEqual(dict(var.__dict__), actual)
def insert_weighted(index_path, dst_wd, dst_master_path):
"""
Inserted weighted, destination variable data into the master destination file.
:param str index_path: Path to the split index netCDF file.
:param str dst_wd: Working directory containing the destination files holding the weighted data.
:param str dst_master_path: Path to the destination master file.
"""
index_field = RequestDataset(index_path).get()
gs_index_v = index_field[GridSplitterConstants.IndexFile.NAME_INDEX_VARIABLE]
dst_filenames = gs_index_v.attrs[GridSplitterConstants.IndexFile.NAME_DESTINATION_VARIABLE]
dst_filenames = index_field[dst_filenames]
y_bounds = GridSplitterConstants.IndexFile.NAME_Y_BOUNDS_VARIABLE
y_bounds = gs_index_v.attrs[y_bounds]
y_bounds = index_field[y_bounds].get_value()
x_bounds = GridSplitterConstants.IndexFile.NAME_X_BOUNDS_VARIABLE
x_bounds = gs_index_v.attrs[x_bounds]
x_bounds = index_field[x_bounds].get_value()
joined = dst_filenames.join_string_value()
dst_master_field = RequestDataset(dst_master_path).get()
for data_variable in dst_master_field.data_variables:
assert data_variable.ndim == 3
assert not data_variable.has_allocated_value
for time_index in range(dst_master_field.time.shape[0]):
for vidx, source_path in enumerate(joined):
source_path = os.path.join(dst_wd, source_path)
slc = {dst_master_field.time.dimensions[0].name: time_index,
dst_master_field.y.dimensions[0].name: slice(None),
dst_master_field.x.dimensions[0].name: slice(None)}
source_data = RequestDataset(source_path).get()[data_variable.name][slc]
assert not source_data.has_allocated_value
with nc_scope(dst_master_path, 'a') as ds:
ds.variables[data_variable.name][time_index, y_bounds[vidx][0]:y_bounds[vidx][1],
x_bounds[vidx][0]:x_bounds[vidx][1]] = source_data.get_value()
