def test_missing_calendar_attribute(self):
# # path to the test data file
out_nc = os.path.join(self._test_dir, self.fn)
## case of calendar being set to an empty string
with nc_scope(out_nc, 'a') as ds:
ds.variables['time'].calendar = ''
rd = ocgis.RequestDataset(uri=out_nc, variable='foo')
## the default for the calendar overload is standard
self.assertEqual(rd.t_calendar, None)
## case of a calendar being set a bad value but read anyway
with nc_scope(out_nc, 'a') as ds:
ds.variables['time'].calendar = 'foo'
rd = ocgis.RequestDataset(uri=out_nc, variable='foo')
field = rd.get()
self.assertEqual(field.temporal.calendar, 'foo')
## calendar is only access when the float values are converted to datetime
## objects
with self.assertRaises(ValueError):
field.temporal.value_datetime
## now overload the value and ensure the field datetimes may be loaded
rd = ocgis.RequestDataset(uri=out_nc, variable='foo', t_calendar='standard')
self.assertEqual(rd.source_metadata['variables']['time']['attrs']['calendar'], 'foo')
field = rd.get()
self.assertEqual(field.temporal.calendar, 'standard')
field.temporal.value_datetime
## case of a missing calendar attribute altogether
with nc_scope(out_nc, 'a') as ds:
ds.variables['time'].delncattr('calendar')
rd = ocgis.RequestDataset(uri=out_nc, variable='foo')
self.assertEqual(rd.t_calendar, None)
self.assertIsInstance(rd.inspect_as_dct(), OrderedDict)
self.assertEqual(rd.inspect_as_dct()['derived']['Calendar'],
'None (will assume "standard")')
## write the data to a netCDF and ensure the calendar is written.
ret = ocgis.OcgOperations(dataset=rd, output_format='nc').execute()
with nc_scope(ret) as ds:
self.assertEqual(ds.variables['time'].calendar, 'standard')
self.assertEqual(ds.variables['time_bnds'].calendar, 'standard')
field = rd.get()
## the standard calendar name should be available at the dataset level
self.assertEqual(field.temporal.calendar, 'standard')
## test different forms of inspect ensuring the standard calendar is
## correctly propagated
ips = [ocgis.Inspect(request_dataset=rd), ocgis.Inspect(uri=out_nc, variable='foo')]
for ip in ips:
self.assertNotIn('calendar', ip.meta['variables']['time']['attrs'])
self.assertTrue(ip.get_temporal_report()[2].endswith(('will assume "standard")')))
ip = ocgis.Inspect(uri=out_nc)
## this method is only applicable when a variable is present
with self.assertRaises(AttributeError):
ip.get_report()
self.assertIsInstance(ip.get_report_no_variable(), list)
def get_multiple_variable_request_dataset_dictionary(self):
rd_orig = self.test_data.get_rd('cancm4_tas')
dest_uri = os.path.join(self._test_dir, os.path.split(rd_orig.uri)[1])
shutil.copy2(rd_orig.uri, dest_uri)
with nc_scope(dest_uri, 'a') as ds:
var = ds.variables['tas']
outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions)
outvar[:] = var[:] + 3
outvar.setncatts(var.__dict__)
with nc_scope(dest_uri) as ds:
self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys())))
return {'uri': dest_uri, 'variable': ['tas', 'tasmax']}
def test_get_field_nonequivalent_units_in_source_data(self):
new_path = self.test_data.copy_file('cancm4_tas', self._test_dir)
## put non-equivalent units on the source data and attempto to conform
with nc_scope(new_path, 'a') as ds:
ds.variables['tas'].units = 'coulomb'
rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius')
with self.assertRaises(RequestValidationError):
rd.get()
## remove units altogether
with nc_scope(new_path, 'a') as ds:
ds.variables['tas'].delncattr('units')
rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius')
with self.assertRaises(NoUnitsError):
rd.get()
def test_init(self):
dataset = self.get_dataset()
uri = dataset['uri']
variable = dataset['variable']
with nc_scope(uri) as ds:
nc_metadata = NcMetadata(ds)
keywords = dict(
uri=[None, self.get_dataset()['uri']],
variable=[None, self.get_dataset()['variable']],
request_dataset=[None, RequestDataset(uri=uri, variable=variable)],
meta=[None, nc_metadata])
for k in itr_products_keywords(keywords, as_namedtuple=True):
try:
ip = Inspect(**k._asdict())
except ValueError:
if k.uri is None and k.request_dataset is None and k.meta is None:
continue
else:
raise
ret = ip.__repr__()
search = re.search('URI = (.*)\n', ret).groups()[0]
if k.uri is None and k.meta is not None and k.request_dataset is None:
self.assertEqual(search, 'None')
else:
self.assertTrue(os.path.exists(search))
def test_get_lines(self):
# test with a unicode string
path = os.path.join(self.current_dir_output, 'foo.nc')
with nc_scope(path, 'w') as ds:
ds.foo = u'a bad \u2013 unicode character'
md = NcMetadata(rootgrp=ds)
ds.sync()
lines = md.get_lines()
self.assertEqual(lines[4], '// global attributes:')
def test_load_dtype_on_dimensions(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
with nc_scope(rd.uri) as ds:
test_dtype_temporal = ds.variables['time'].dtype
test_dtype_value = ds.variables['tas'].dtype
self.assertEqual(field.temporal.dtype,test_dtype_temporal)
self.assertEqual(field.variables['tas'].dtype,test_dtype_value)
self.assertEqual(field.temporal.dtype,np.float64)
def test_to_netcdf_with_geometry(self):
rd = self.test_data.get_rd('narccap_rotated_pole')
## this bounding box covers the entire spatial domain. the software will
## move between rotated pole and CFWGS84 using this operation. it can then
## be compared against the "null" result which just does a snippet.
geom = [-173.3,8.8,-20.6,79.0]
ops = OcgOperations(dataset=rd,output_format='nc',snippet=True,geom=geom)
ret = ops.execute()
ops2 = OcgOperations(dataset=rd,output_format='nc',snippet=True,prefix='hi')
ret2 = ops2.execute()
self.assertNcEqual(ret,ret2,metadata_only=True,ignore_attributes={'global': ['history']})
with nc_scope(ret) as ds:
with nc_scope(ret2) as ds2:
for var_name in ['yc','xc','tas']:
var = ds.variables[var_name][:]
var2 = ds2.variables[var_name][:]
diff = np.abs(var-var2)
self.assertTrue(diff.max() <= 1.02734374963e-06)
def test_read_write_projections(self):
data_dir = '/usr/local/climate_data/narccap'
ocgis.env.DIR_DATA = data_dir
ocgis.env.OVERWRITE = True
real = {'pr': {'pr_RCM3_cgcm3_1986010103.nc': {'mu': 2.7800052478033606e-07, 'shape': (1, 1, 1, 7, 15)}, 'pr_MM5I_ncep_1981010103.nc': {'mu': 3.3648159627007675e-08, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_ncep_1986010103.nc': {'mu': 9.7176247154926553e-09, 'shape': (1, 1, 1, 7, 15)}, 'pr_CRCM_ncep_1986010103.nc': {'mu': 1.1799650910219663e-26, 'shape': (1, 1, 1, 8, 16)}, 'pr_CRCM_cgcm3_1981010103.nc': {'mu': 2.6299784818262446e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_MM5I_ncep_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_HRM3_ncep_1981010103.nc': {'mu': 5.507401147596971e-10, 'shape': (1, 1, 1, 31, 22)}, 'pr_RCM3_cgcm3_1981010103.nc': {'mu': 1.18896825283411e-05, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_gfdl_1986010100.nc': {'mu': 2.0890602963450161e-07, 'shape': (1, 1, 1, 7, 15)}, 'pr_WRFG_cgcm3_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_ECP2_gfdl_1981010103.nc': {'mu': 6.1180394635919263e-06, 'shape': (1, 1, 1, 9, 17)}, 'pr_CRCM_ncep_1981010103.nc': {'mu': 2.767125774613198e-05, 'shape': (1, 1, 1, 8, 16)}, 'pr_HRM3_gfdl_1986010103.nc': {'mu': 4.1377767579766305e-06, 'shape': (1, 1, 1, 31, 22)}, 'pr_RCM3_gfdl_1981010103.nc': {'mu': -5.1954553518551086e-24, 'shape': (1, 1, 1, 7, 15)}, 'pr_HRM3_ncep_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 31, 22)}, 'pr_TMSL_ccsm_1986010103.nc': {'mu': 3.734873736402074e-07, 'shape': (1, 1, 1, 7, 14)}, 'pr_HRM3_gfdl_1981010103.nc': {'mu': 5.2488248024374339e-07, 'shape': (1, 1, 1, 31, 22)}, 'pr_WRFG_ccsm_1986010103.nc': {'mu': 0.00010390303979970907, 'shape': (1, 1, 1, 7, 14)}, 'pr_MM5I_ccsm_1986010103.nc': {'mu': 5.0342728890858494e-07, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_ccsm_1981010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_cgcm3_1981010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_ncep_1981010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_ncep_1981010103.nc': {'mu': 7.637150009118376e-06, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_ccsm_1981010103.nc': {'mu': 9.641077844117023e-27, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_gfdl_1986010103.nc': {'mu': 1.0929620614097941e-05, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_gfdl_1981010100.nc': {'mu': 1.3174895956811014e-10, 'shape': (1, 1, 1, 7, 15)}, 'pr_CRCM_ccsm_1981010103.nc': {'mu': 1.6264247653238914e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_WRFG_ncep_1986010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_CRCM_cgcm3_1986010103.nc': {'mu': 3.152432755621917e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_MM5I_ccsm_1981010103.nc': {'mu': 1.5723979779044096e-09, 'shape': (1, 1, 1, 7, 14)}, 'pr_CRCM_ccsm_1986010103.nc': {'mu': 1.1736681164406678e-05, 'shape': (1, 1, 1, 8, 16)}, 'pr_ECP2_gfdl_1986010103.nc': {'mu': 9.865492043614972e-06, 'shape': (1, 1, 1, 9, 17)}}, 'tas': {'tas_TMSL_gfdl_1986010100.nc': {'mu': 272.1787109375, 'shape': (1, 1, 1, 7, 15)}, 'tas_RCM3_gfdl_1986010103.nc': {'mu': 257.74983723958331, 'shape': (1, 1, 1, 7, 15)}, 'tas_HRM3_ncep_1986010103.nc': {'mu': 272.10732660060978, 'shape': (1, 1, 1, 31, 22)}, 'tas_WRFG_ccsm_1981010103.nc': {'mu': 259.1943359375, 'shape': (1, 1, 1, 7, 14)}, 'tas_TMSL_ccsm_1986010103.nc': {'mu': 271.766502490942, 'shape': (1, 1, 1, 7, 14)}, 'tas_CRCM_cgcm3_1981010103.nc': {'mu': 256.05007276348039, 'shape': (1, 1, 1, 8, 16)}, 'tas_RCM3_ncep_1981010103.nc': {'mu': 275.49927920386904, 'shape': (1, 1, 1, 7, 15)}, 'tas_RCM3_gfdl_1981010103.nc': {'mu': 264.29543340773807, 'shape': (1, 1, 1, 7, 15)}, 'tas_CRCM_ncep_1986010103.nc': {'mu': 268.38143382352939, 'shape': (1, 1, 1, 8, 16)}, 'tas_CRCM_cgcm3_1986010103.nc': {'mu': 271.96783088235293, 'shape': (1, 1, 1, 8, 16)}, 'tas_CRCM_ccsm_1986010103.nc': {'mu': 262.36866191789215, 'shape': (1, 1, 1, 8, 16)}, 'tas_WRFG_cgcm3_1986010103.nc': {'mu': 274.17369962993422, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ccsm_1986010103.nc': {'mu': 260.47268194901318, 'shape': (1, 1, 1, 7, 14)}, 'tas_TMSL_ccsm_1981010103.nc': {'mu': 275.4296875, 'shape': (1, 1, 1, 7, 14)}, 'tas_WRFG_ccsm_1986010103.nc': {'mu': 260.7568359375, 'shape': (1, 1, 1, 7, 14)}, 'tas_RCM3_ncep_1986010103.nc': {'mu': 268.04431733630952, 'shape': (1, 1, 1, 7, 15)}, 'tas_CRCM_ncep_1981010103.nc': {'mu': 273.35757506127453, 'shape': (1, 1, 1, 8, 16)}, 'tas_ECP2_gfdl_1981010103.nc': {'mu': 261.00524662990193, 'shape': (1, 1, 1, 9, 17)}, 'tas_HRM3_ncep_1981010103.nc': {'mu': 274.20317263719511, 'shape': (1, 1, 1, 31, 22)}, 'tas_RCM3_cgcm3_1981010103.nc': {'mu': 270.27541387648807, 'shape': (1, 1, 1, 7, 15)}, 'tas_TMSL_gfdl_1981010100.nc': {'mu': 274.62939453125, 'shape': (1, 1, 1, 7, 15)}, 'tas_HRM3_gfdl_1981010103.nc': {'mu': 257.89143483231709, 'shape': (1, 1, 1, 31, 22)}, 'tas_WRFG_cgcm3_1981010103.nc': {'mu': 266.04286595394734, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ccsm_1981010103.nc': {'mu': 266.87697882401318, 'shape': (1, 1, 1, 7, 14)}, 'tas_WRFG_ncep_1981010103.nc': {'mu': 275.5278834292763, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ncep_1981010103.nc': {'mu': 273.7758275082237, 'shape': (1, 1, 1, 7, 14)}, 'tas_ECP2_gfdl_1986010103.nc': {'mu': 267.40525428921569, 'shape': (1, 1, 1, 9, 17)}, 'tas_MM5I_ncep_1986010103.nc': {'mu': 267.03638980263156, 'shape': (1, 1, 1, 7, 14)}, 'tas_RCM3_cgcm3_1986010103.nc': {'mu': 272.37116350446428, 'shape': (1, 1, 1, 7, 15)}, 'tas_WRFG_ncep_1986010103.nc': {'mu': 268.89250102796052, 'shape': (1, 1, 1, 7, 14)}, 'tas_HRM3_gfdl_1986010103.nc': {'mu': 270.4895674542683, 'shape': (1, 1, 1, 31, 22)}, 'tas_CRCM_ccsm_1981010103.nc': {'mu': 264.73590686274508, 'shape': (1, 1, 1, 8, 16)}}}
for uri in os.listdir(data_dir):
if uri.endswith('nc'):
variable = uri.split('_')[0]
for output_format in [
'numpy',
'nc'
]:
rd = RequestDataset(uri=uri,variable=variable)#,time_range=time_range)
try:
ops = OcgOperations(dataset=rd,geom='state_boundaries',select_ugid=[16],snippet=True,
output_format=output_format)
except DefinitionValidationError:
## rotated pole may not be written to netCDF
crs = rd._get_crs_()
if isinstance(crs,CFRotatedPole):
continue
else:
raise
try:
ret = ops.execute()
if output_format == 'numpy':
ref = ret[16].values()[0].variables.values()[0].value
mu = np.ma.mean(ref)
r_mu = real[variable][uri]['mu']
try:
self.assertAlmostEqual(r_mu,mu)
except AssertionError:
self.assertEqual(r_mu,np.ma.core.MaskedConstant)
self.assertEqual(real[variable][uri]['shape'],ref.shape)
if output_format == 'nc':
with nc_scope(ret) as ds:
try:
grid_mapping = ds.variables[rd.variable].grid_mapping
self.assertTrue(grid_mapping in ds.variables)
except AttributeError:
## time slice files do not have a projection
self.assertTrue('_TMSL_' in rd.uri)
except ExtentError:
if 'ECP2_ncep' in rd.uri:
pass
else:
raise
def test_get_dimensioned_variables_two_variables_in_target_dataset(self):
rd_orig = self.test_data.get_rd('cancm4_tas')
dest_uri = os.path.join(self._test_dir, os.path.split(rd_orig.uri)[1])
shutil.copy2(rd_orig.uri, dest_uri)
with nc_scope(dest_uri, 'a') as ds:
var = ds.variables['tas']
outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions)
outvar[:] = var[:] + 3
outvar.setncatts(var.__dict__)
rd = RequestDataset(uri=dest_uri)
self.assertEqual(rd.variable, ('tas', 'tasmax'))
self.assertEqual(rd.variable, rd.alias)
def test_selecting_single_value(self):
rd = self.test_data.get_rd('cancm4_tas')
lat_index = 32
lon_index = 97
with nc_scope(rd.uri) as ds:
lat_value = ds.variables['lat'][lat_index]
lon_value = ds.variables['lon'][lon_index]
data_values = ds.variables['tas'][:,lat_index,lon_index]
ops = ocgis.OcgOperations(dataset=rd,geom=[lon_value,lat_value],search_radius_mult=0.1)
ret = ops.execute()
values = np.squeeze(ret[1]['tas'].variables['tas'].value)
self.assertNumpyAll(data_values,values)
geom = Point(lon_value,lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd,geom=geom)
ret = ops.execute()
values = np.squeeze(ret[1]['tas'].variables['tas'].value)
self.assertNumpyAll(data_values,values)
geom = Point(lon_value-360.,lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd,geom=geom)
ret = ops.execute()
values = np.squeeze(ret[1]['tas'].variables['tas'].value)
self.assertNumpyAll(data_values,values)
geom = Point(lon_value-360.,lat_value).buffer(0.001)
ops = ocgis.OcgOperations(dataset=rd,geom=geom,aggregate=True,spatial_operation='clip')
ret = ops.execute()
values = np.squeeze(ret[1]['tas'].variables['tas'].value)
self.assertNumpyAll(data_values,values)
ops = ocgis.OcgOperations(dataset=rd,geom=[lon_value,lat_value],
search_radius_mult=0.1,output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
values = np.squeeze(ds.variables['tas'][:])
self.assertNumpyAll(data_values,values)
def test_calculate_opendap(self):
## test against an opendap target ensuring icclim and ocgis operations
## are equivalent in the netcdf output
url = 'http://opendap.nmdc.eu/knmi/thredds/dodsC/IS-ENES/TESTSETS/tasmax_day_EC-EARTH_rcp26_r8i1p1_20760101-21001231.nc'
calc_grouping = ['month']
rd = ocgis.RequestDataset(uri=url,variable='tasmax')
calc_icclim = [{'func':'icclim_SU','name':'SU'}]
ops = ocgis.OcgOperations(dataset=rd,calc=calc_icclim,calc_grouping=calc_grouping,
output_format='nc',geom='state_boundaries',select_ugid=[10],
prefix='icclim')
ret_icclim = ops.execute()
calc_ocgis = [{'func':'threshold','name':'SU','kwds':{'threshold':298.15,'operation':'gt'}}]
ops = ocgis.OcgOperations(dataset=rd,calc=calc_ocgis,calc_grouping=calc_grouping,
output_format='nc',geom='state_boundaries',select_ugid=[10],
prefix='ocgis')
ret_ocgis = ops.execute()
## variable and datasets will have different attributes, so adjust those
## before testing if the netCDFs are equal...
with nc_scope(ret_icclim,'r') as ds_icclim:
with nc_scope(ret_ocgis,'a') as ds_ocgis:
## strip the current attributes
for key in ds_ocgis.ncattrs():
ds_ocgis.delncattr(key)
for key in ds_ocgis.variables['SU'].ncattrs():
if not key.startswith('_'):
ds_ocgis.variables['SU'].delncattr(key)
## make equivalent attributes
for key in ds_icclim.ncattrs():
setattr(ds_ocgis,key,getattr(ds_icclim,key))
## update the target variable attributes
for key,value in ds_icclim.variables['SU'].__dict__.iteritems():
if not key.startswith('_'):
setattr(ds_ocgis.variables['SU'],key,value)
self.assertNcEqual(ret_icclim,ret_ocgis)
def test_with_overloads_real_data(self):
## copy the test file as the calendar attribute will be modified
rd = self.test_data.get_rd('cancm4_tas')
filename = os.path.split(rd.uri)[1]
dest = os.path.join(self._test_dir, filename)
shutil.copy2(rd.uri, dest)
## modify the calendar attribute
with nc_scope(dest, 'a') as ds:
self.assertEqual(ds.variables['time'].calendar, '365_day')
ds.variables['time'].calendar = '365_days'
## assert the calendar is in fact changed on the source file
with nc_scope(dest, 'r') as ds:
self.assertEqual(ds.variables['time'].calendar, '365_days')
rd2 = RequestDataset(uri=dest, variable='tas')
field = rd2.get()
## the bad calendar will raise a value error when the datetimes are
## converted.
with self.assertRaises(ValueError):
field.temporal.value_datetime
## overload the calendar and confirm the datetime values are the same
## as the datetime values from the original good file
rd3 = RequestDataset(uri=dest, variable='tas', t_calendar='365_day')
field = rd3.get()
self.assertNumpyAll(field.temporal.value_datetime, rd.get().temporal.value_datetime)
## pass as a dataset collection to operations and confirm the data may
## be written to a flat file. dates are converted in the process.
time_range = (datetime.datetime(2001, 1, 1, 0, 0), datetime.datetime(2011, 1, 1, 0, 0))
dataset = [{'time_region': None,
'uri': dest,
'time_range': time_range,
'alias': u'tas',
't_units': u'days since 1850-1-1',
'variable': u'tas',
't_calendar': u'365_day'}]
rdc = RequestDatasetCollection(dataset)
ops = OcgOperations(dataset=rdc, geom='state_boundaries', select_ugid=[25], output_format='csv+')
ops.execute()
def test_operations(self):
uri = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(uri=uri,
variable='tas',
# time_range=[datetime.datetime(2001,1,1),datetime.datetime(2003,12,31,23,59)]
)
calc_grouping = ['month','year']
calc = [{'func':'dynamic_kernel_percentile_threshold','name':'tg10p','kwds':{'percentile':10,'width':5,'operation':'lt'}}]
ops = OcgOperations(dataset=rd,calc_grouping=calc_grouping,calc=calc,
output_format='nc')
ret = ops.execute()
with nc_scope(ret) as ds:
ref = ds.variables['tg10p'][:]
self.assertAlmostEqual(ref.mean(),2.9763946533203125)
def test_calculation_operations_to_nc(self):
rd = self.test_data.get_rd('cancm4_tasmax_2011')
slc = [None,None,None,[0,10],[0,10]]
ops_ocgis = OcgOperations(calc=[{'func':'icclim_SU','name':'SU'}],
calc_grouping=['month'],
slice=slc,
dataset=rd,
output_format='nc')
ret = ops_ocgis.execute()
with nc_scope(ret) as ds:
to_test = deepcopy(ds.__dict__)
history = to_test.pop('history')
self.assertEqual(history[111:187],' Calculation of SU indice (monthly climatology) from 2011-1-1 to 2020-12-31.')
self.assertDictEqual(to_test,OrderedDict([(u'source_data_global_attributes', u'{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2010", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "64384802-3f0f-4ab4-b569-697bd5430854", "branch_time_YMDH": "2011:01:01:00", "CCCma_runid": "DHFP1B_E002_I2011_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2010", "frequency": "day", "creation_date": "2012-03-28T15:32:08Z", "history": "2012-03-28T15:32:08Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2010", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.8.0"}'), (u'title', u'ECA heat indice SU'), (u'references', u'ATBD of the ECA indices calculation (http://eca.knmi.nl/documents/atbd.pdf)'), (u'institution', u'Climate impact portal (http://climate4impact.eu)'), (u'comment', u' ')]))
var = ds.variables['SU']
to_test = dict(var.__dict__)
self.assertEqual(to_test,{'_FillValue':999999,u'units': u'days', u'standard_name': AbstractIcclimFunction.standard_name, u'long_name': 'Summer days (number of days where daily maximum temperature > 25 degrees)'})
def test_calculate_rotated_pole(self):
tasmin_fake = self.test_data.get_rd('rotated_pole_ichec')
tasmin_fake.alias = 'tasmin'
tasmax_fake = deepcopy(tasmin_fake)
tasmax_fake.alias = 'tasmax'
rds = [tasmin_fake,tasmax_fake]
for rd in rds:
rd.time_region = {'year':[1973]}
calc_ETR = [{'func':'icclim_ETR','name':'ETR','kwds':{'tasmin':'tasmin','tasmax':'tasmax'}}]
ops = ocgis.OcgOperations(dataset=[tasmin_fake,tasmax_fake],
calc=calc_ETR,
calc_grouping=['year', 'month'],
prefix = 'ETR_ocgis_icclim',
output_format = 'nc',
add_auxiliary_files=False)
with nc_scope(ops.execute()) as ds:
self.assertEqual(ds.variables['ETR'][:].shape,(12,103,106))
def test_standard_AbstractIcclimFunction(self):
shapes = ([('month',), 12],[('month', 'year'), 24],[('year',),2])
ocgis.env.OVERWRITE = True
keys = set(library_icclim._icclim_function_map.keys())
for klass in [
AbstractIcclimUnivariateSetFunction,
AbstractIcclimMultivariateFunction]:
for subclass in itersubclasses(klass):
keys.remove(subclass.key)
self.assertEqual([('month',),('month','year'),('year',)],subclass._allowed_temporal_groupings)
for cg in CalcGrouping.iter_possible():
calc = [{'func':subclass.key,'name':subclass.key.split('_')[1]}]
if klass == AbstractIcclimUnivariateSetFunction:
rd = self.test_data.get_rd('cancm4_tas')
rd.time_region = {'year':[2001,2002]}
calc = [{'func':subclass.key,'name':subclass.key.split('_')[1]}]
else:
tasmin = self.test_data.get_rd('cancm4_tasmin_2001')
tasmax = self.test_data.get_rd('cancm4_tasmax_2001')
rd = [tasmin,tasmax]
for r in rd:
r.time_region = {'year':[2001,2002]}
calc[0].update({'kwds':{'tasmin':'tasmin','tasmax':'tasmax'}})
try:
ops = ocgis.OcgOperations(dataset=rd,
output_format='nc',
calc=calc,
calc_grouping=cg,
geom=[3.39,40.62,10.54,52.30])
ret = ops.execute()
to_test = None
for shape in shapes:
if shape[0] == cg:
to_test = shape[1]
with nc_scope(ret) as ds:
var = ds.variables[calc[0]['name']]
self.assertEqual(var.dtype,subclass.dtype)
self.assertEqual(var.shape,(to_test,5,4))
except DefinitionValidationError as e:
if e.message.startswith('''OcgOperations validation raised an exception on the argument/operation "calc_grouping" with the message: The following temporal groupings are supported for ICCLIM: [('month',), ('month', 'year'), ('year',)]. The requested temporal group is:'''):
pass
else:
raise(e)
self.assertEqual(len(keys),0)
def test_calculation_operations_to_nc(self):
rd = self.test_data.get_rd('cancm4_tas')
slc = [None,None,None,[0,10],[0,10]]
ops_ocgis = OcgOperations(calc=[{'func':'icclim_TG','name':'TG'}],
calc_grouping=['month'],
slice=slc,
dataset=rd,
output_format='nc')
ret = ops_ocgis.execute()
with nc_scope(ret) as ds:
self.assertIn('Calculation of TG indice (monthly climatology)',ds.history)
self.assertEqual(ds.title,'ECA temperature indice TG')
var = ds.variables['TG']
## check the JSON serialization
self.assertEqual(ds.__dict__[AbstractIcclimFunction._global_attribute_source_name],
u'{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2000", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "fac7bd83-dd7a-425b-b4dc-b5ab2e915939", "branch_time_YMDH": "2001:01:01:00", "CCCma_runid": "DHFP1B_E002_I2001_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2000", "frequency": "day", "creation_date": "2011-05-08T01:01:51Z", "history": "2011-05-08T01:01:51Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2000", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.5.4"}')
## load the original source attributes from the JSON string
json.loads(ds.__dict__[AbstractIcclimFunction._global_attribute_source_name])
self.assertEqual(dict(var.__dict__),{'_FillValue':np.float32(1e20),u'units': u'K', u'standard_name': AbstractIcclimFunction.standard_name, u'long_name': u'Mean of daily mean temperature'})
def test_operations(self):
uri = self.test_data.get_uri("cancm4_tas")
rd = RequestDataset(
uri=uri,
variable="tas",
# time_range=[datetime.datetime(2001,1,1),datetime.datetime(2003,12,31,23,59)]
)
calc_grouping = ["month", "year"]
calc = [
{
"func": "dynamic_kernel_percentile_threshold",
"name": "tg10p",
"kwds": {"percentile": 10, "width": 5, "operation": "lt"},
}
]
ops = OcgOperations(dataset=rd, calc_grouping=calc_grouping, calc=calc, output_format="nc")
ret = ops.execute()
with nc_scope(ret) as ds:
ref = ds.variables["tg10p_tas"][:]
self.assertAlmostEqual(ref.mean(), 2.9763946533203125)
def test_multiple_variables(self):
conv_klasses = [CsvConverter, NcConverter]
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
var2 = deepcopy(field.variables['tas'])
var2.alias = 'tas2'
field.variables.add_variable(deepcopy(var2), assign_new_uid=True)
field = field[:, 0:2, :, 0:5, 0:5]
coll = self.get_spatial_collection(field=field)
for conv_klass in conv_klasses:
conv = conv_klass([coll], self._test_dir, 'ocgis_output_{0}'.format(conv_klass.__name__))
ret = conv.write()
if conv_klass == CsvConverter:
with open(ret, 'r') as f:
reader = DictReader(f)
aliases = set([row['ALIAS'] for row in reader])
self.assertEqual(set(['tas', 'tas2']), aliases)
else:
with nc_scope(ret) as ds:
self.assertAlmostEqual(ds.variables['tas'][:].mean(), 247.08416015624999)
self.assertNumpyAll(ds.variables['tas'][:], ds.variables['tas2'][:])
def test_init(self):
with nc_scope(self.rd.uri, 'r') as ds:
ncm = NcMetadata(ds)
self.assertEqual(set(ncm.keys()), set(['dataset', 'variables', 'dimensions', 'file_format']))
def test_init_combinations(self):
rd_orig = self.test_data.get_rd('cancm4_tas')
dest_uri = os.path.join(self._test_dir, os.path.split(rd_orig.uri)[1])
shutil.copy2(rd_orig.uri, dest_uri)
with nc_scope(dest_uri, 'a') as ds:
var = ds.variables['tas']
outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions)
outvar[:] = var[:] + 3
outvar.setncatts(var.__dict__)
with nc_scope(dest_uri) as ds:
self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys())))
keywords = dict(
name=[None, 'foo'],
uri=[None, dest_uri],
variable=[None, 'tas', ['tas', 'tasmax'], 'crap'],
alias=[None, 'tas', ['tas', 'tasmax'], ['tas_alias', 'tasmax_alias']],
units=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy']],
conform_units_to=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy'], [None, 'coulomb'], ['coulomb', 'coulomb']])
def itr_row(key, sequence):
for element in sequence:
yield ({key: element})
def itr_products_keywords(keywords):
iterators = [itr_row(ki, vi) for ki, vi in keywords.iteritems()]
for dictionaries in itertools.product(*iterators):
yld = {}
for dictionary in dictionaries:
yld.update(dictionary)
yield yld
for k in itr_products_keywords(keywords):
try:
rd = RequestDataset(**k)
self.assertEqual(rd._source_metadata, None)
self.assertEqual(len(rd._variable), len(rd._units))
if k['name'] is None:
self.assertEqual(rd.name, '_'.join(rd._alias))
else:
self.assertEqual(rd.name, 'foo')
for v in rd._variable:
try:
self.assertTrue(v in rd.source_metadata['variables'].keys())
except VariableNotFoundError:
if 'crap' in rd._variable:
self.assertEqual(rd._source_metadata, None)
break
if k['units'] is None and len(rd._variable) == 1:
self.assertEqual(rd.units, None)
self.assertEqual(rd._units, (None,))
try:
field = rd.get()
self.assertEqual(field.name, rd.name)
self.assertEqual(set(field.variables.keys()), set(rd._alias))
except VariableNotFoundError:
if 'crap' in rd._variable:
continue
else:
raise
except RequestValidationError:
if 'coulomb' in get_tuple(k['conform_units_to']):
continue
else:
raise
except RequestValidationError as e:
## uris cannot be None
if k['uri'] is None:
pass
## variables cannot be None
elif k['variable'] is None:
pass
## 'crap' is not a real variable name
elif k['conform_units_to'] is not None and (k['conform_units_to'] == 'crap' or \
'crap' in k['conform_units_to']):
pass
## conform_units_to must match units element-wise
elif k['conform_units_to'] is not None and k['variable'] is not None and \
len(k['conform_units_to']) != len(k['variable']):
pass
## aliases must occur for each variable
elif len(get_tuple(k['alias'])) != len(get_tuple(k['variable'])):
pass
## units must occur for each variable
elif len(get_tuple(k['units'])) != len(get_tuple(k['variable'])):
pass
## bad unit definition
## 'crap' is not a real variable name
elif k['units'] is not None and (k['units'] == 'crap' or \
'crap' in k['units']):
pass
## alway need a uri and variable
elif k['uri'] is None:
pass
else:
print k
raise
except:
print k
raise
