def test_missing_calendar_attribute(self): # # path to the test data file out_nc = os.path.join(self._test_dir, self.fn) ## case of calendar being set to an empty string with nc_scope(out_nc, 'a') as ds: ds.variables['time'].calendar = '' rd = ocgis.RequestDataset(uri=out_nc, variable='foo') ## the default for the calendar overload is standard self.assertEqual(rd.t_calendar, None) ## case of a calendar being set a bad value but read anyway with nc_scope(out_nc, 'a') as ds: ds.variables['time'].calendar = 'foo' rd = ocgis.RequestDataset(uri=out_nc, variable='foo') field = rd.get() self.assertEqual(field.temporal.calendar, 'foo') ## calendar is only access when the float values are converted to datetime ## objects with self.assertRaises(ValueError): field.temporal.value_datetime ## now overload the value and ensure the field datetimes may be loaded rd = ocgis.RequestDataset(uri=out_nc, variable='foo', t_calendar='standard') self.assertEqual(rd.source_metadata['variables']['time']['attrs']['calendar'], 'foo') field = rd.get() self.assertEqual(field.temporal.calendar, 'standard') field.temporal.value_datetime ## case of a missing calendar attribute altogether with nc_scope(out_nc, 'a') as ds: ds.variables['time'].delncattr('calendar') rd = ocgis.RequestDataset(uri=out_nc, variable='foo') self.assertEqual(rd.t_calendar, None) self.assertIsInstance(rd.inspect_as_dct(), OrderedDict) self.assertEqual(rd.inspect_as_dct()['derived']['Calendar'], 'None (will assume "standard")') ## write the data to a netCDF and ensure the calendar is written. ret = ocgis.OcgOperations(dataset=rd, output_format='nc').execute() with nc_scope(ret) as ds: self.assertEqual(ds.variables['time'].calendar, 'standard') self.assertEqual(ds.variables['time_bnds'].calendar, 'standard') field = rd.get() ## the standard calendar name should be available at the dataset level self.assertEqual(field.temporal.calendar, 'standard') ## test different forms of inspect ensuring the standard calendar is ## correctly propagated ips = [ocgis.Inspect(request_dataset=rd), ocgis.Inspect(uri=out_nc, variable='foo')] for ip in ips: self.assertNotIn('calendar', ip.meta['variables']['time']['attrs']) self.assertTrue(ip.get_temporal_report()[2].endswith(('will assume "standard")'))) ip = ocgis.Inspect(uri=out_nc) ## this method is only applicable when a variable is present with self.assertRaises(AttributeError): ip.get_report() self.assertIsInstance(ip.get_report_no_variable(), list)
def get_multiple_variable_request_dataset_dictionary(self): rd_orig = self.test_data.get_rd('cancm4_tas') dest_uri = os.path.join(self._test_dir, os.path.split(rd_orig.uri)[1]) shutil.copy2(rd_orig.uri, dest_uri) with nc_scope(dest_uri, 'a') as ds: var = ds.variables['tas'] outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions) outvar[:] = var[:] + 3 outvar.setncatts(var.__dict__) with nc_scope(dest_uri) as ds: self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys()))) return {'uri': dest_uri, 'variable': ['tas', 'tasmax']}
def test_get_field_nonequivalent_units_in_source_data(self): new_path = self.test_data.copy_file('cancm4_tas', self._test_dir) ## put non-equivalent units on the source data and attempto to conform with nc_scope(new_path, 'a') as ds: ds.variables['tas'].units = 'coulomb' rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius') with self.assertRaises(RequestValidationError): rd.get() ## remove units altogether with nc_scope(new_path, 'a') as ds: ds.variables['tas'].delncattr('units') rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius') with self.assertRaises(NoUnitsError): rd.get()
def test_init(self): dataset = self.get_dataset() uri = dataset['uri'] variable = dataset['variable'] with nc_scope(uri) as ds: nc_metadata = NcMetadata(ds) keywords = dict( uri=[None, self.get_dataset()['uri']], variable=[None, self.get_dataset()['variable']], request_dataset=[None, RequestDataset(uri=uri, variable=variable)], meta=[None, nc_metadata]) for k in itr_products_keywords(keywords, as_namedtuple=True): try: ip = Inspect(**k._asdict()) except ValueError: if k.uri is None and k.request_dataset is None and k.meta is None: continue else: raise ret = ip.__repr__() search = re.search('URI = (.*)\n', ret).groups()[0] if k.uri is None and k.meta is not None and k.request_dataset is None: self.assertEqual(search, 'None') else: self.assertTrue(os.path.exists(search))
def test_get_lines(self): # test with a unicode string path = os.path.join(self.current_dir_output, 'foo.nc') with nc_scope(path, 'w') as ds: ds.foo = u'a bad \u2013 unicode character' md = NcMetadata(rootgrp=ds) ds.sync() lines = md.get_lines() self.assertEqual(lines[4], '// global attributes:')
def test_load_dtype_on_dimensions(self): rd = self.test_data.get_rd('cancm4_tas') field = rd.get() with nc_scope(rd.uri) as ds: test_dtype_temporal = ds.variables['time'].dtype test_dtype_value = ds.variables['tas'].dtype self.assertEqual(field.temporal.dtype,test_dtype_temporal) self.assertEqual(field.variables['tas'].dtype,test_dtype_value) self.assertEqual(field.temporal.dtype,np.float64)
def test_to_netcdf_with_geometry(self): rd = self.test_data.get_rd('narccap_rotated_pole') ## this bounding box covers the entire spatial domain. the software will ## move between rotated pole and CFWGS84 using this operation. it can then ## be compared against the "null" result which just does a snippet. geom = [-173.3,8.8,-20.6,79.0] ops = OcgOperations(dataset=rd,output_format='nc',snippet=True,geom=geom) ret = ops.execute() ops2 = OcgOperations(dataset=rd,output_format='nc',snippet=True,prefix='hi') ret2 = ops2.execute() self.assertNcEqual(ret,ret2,metadata_only=True,ignore_attributes={'global': ['history']}) with nc_scope(ret) as ds: with nc_scope(ret2) as ds2: for var_name in ['yc','xc','tas']: var = ds.variables[var_name][:] var2 = ds2.variables[var_name][:] diff = np.abs(var-var2) self.assertTrue(diff.max() <= 1.02734374963e-06)
def test_read_write_projections(self): data_dir = '/usr/local/climate_data/narccap' ocgis.env.DIR_DATA = data_dir ocgis.env.OVERWRITE = True real = {'pr': {'pr_RCM3_cgcm3_1986010103.nc': {'mu': 2.7800052478033606e-07, 'shape': (1, 1, 1, 7, 15)}, 'pr_MM5I_ncep_1981010103.nc': {'mu': 3.3648159627007675e-08, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_ncep_1986010103.nc': {'mu': 9.7176247154926553e-09, 'shape': (1, 1, 1, 7, 15)}, 'pr_CRCM_ncep_1986010103.nc': {'mu': 1.1799650910219663e-26, 'shape': (1, 1, 1, 8, 16)}, 'pr_CRCM_cgcm3_1981010103.nc': {'mu': 2.6299784818262446e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_MM5I_ncep_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_HRM3_ncep_1981010103.nc': {'mu': 5.507401147596971e-10, 'shape': (1, 1, 1, 31, 22)}, 'pr_RCM3_cgcm3_1981010103.nc': {'mu': 1.18896825283411e-05, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_gfdl_1986010100.nc': {'mu': 2.0890602963450161e-07, 'shape': (1, 1, 1, 7, 15)}, 'pr_WRFG_cgcm3_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_ECP2_gfdl_1981010103.nc': {'mu': 6.1180394635919263e-06, 'shape': (1, 1, 1, 9, 17)}, 'pr_CRCM_ncep_1981010103.nc': {'mu': 2.767125774613198e-05, 'shape': (1, 1, 1, 8, 16)}, 'pr_HRM3_gfdl_1986010103.nc': {'mu': 4.1377767579766305e-06, 'shape': (1, 1, 1, 31, 22)}, 'pr_RCM3_gfdl_1981010103.nc': {'mu': -5.1954553518551086e-24, 'shape': (1, 1, 1, 7, 15)}, 'pr_HRM3_ncep_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 31, 22)}, 'pr_TMSL_ccsm_1986010103.nc': {'mu': 3.734873736402074e-07, 'shape': (1, 1, 1, 7, 14)}, 'pr_HRM3_gfdl_1981010103.nc': {'mu': 5.2488248024374339e-07, 'shape': (1, 1, 1, 31, 22)}, 'pr_WRFG_ccsm_1986010103.nc': {'mu': 0.00010390303979970907, 'shape': (1, 1, 1, 7, 14)}, 'pr_MM5I_ccsm_1986010103.nc': {'mu': 5.0342728890858494e-07, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_ccsm_1981010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_cgcm3_1981010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_ncep_1981010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_ncep_1981010103.nc': {'mu': 7.637150009118376e-06, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_ccsm_1981010103.nc': {'mu': 9.641077844117023e-27, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_gfdl_1986010103.nc': {'mu': 1.0929620614097941e-05, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_gfdl_1981010100.nc': {'mu': 1.3174895956811014e-10, 'shape': (1, 1, 1, 7, 15)}, 'pr_CRCM_ccsm_1981010103.nc': {'mu': 1.6264247653238914e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_WRFG_ncep_1986010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_CRCM_cgcm3_1986010103.nc': {'mu': 3.152432755621917e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_MM5I_ccsm_1981010103.nc': {'mu': 1.5723979779044096e-09, 'shape': (1, 1, 1, 7, 14)}, 'pr_CRCM_ccsm_1986010103.nc': {'mu': 1.1736681164406678e-05, 'shape': (1, 1, 1, 8, 16)}, 'pr_ECP2_gfdl_1986010103.nc': {'mu': 9.865492043614972e-06, 'shape': (1, 1, 1, 9, 17)}}, 'tas': {'tas_TMSL_gfdl_1986010100.nc': {'mu': 272.1787109375, 'shape': (1, 1, 1, 7, 15)}, 'tas_RCM3_gfdl_1986010103.nc': {'mu': 257.74983723958331, 'shape': (1, 1, 1, 7, 15)}, 'tas_HRM3_ncep_1986010103.nc': {'mu': 272.10732660060978, 'shape': (1, 1, 1, 31, 22)}, 'tas_WRFG_ccsm_1981010103.nc': {'mu': 259.1943359375, 'shape': (1, 1, 1, 7, 14)}, 'tas_TMSL_ccsm_1986010103.nc': {'mu': 271.766502490942, 'shape': (1, 1, 1, 7, 14)}, 'tas_CRCM_cgcm3_1981010103.nc': {'mu': 256.05007276348039, 'shape': (1, 1, 1, 8, 16)}, 'tas_RCM3_ncep_1981010103.nc': {'mu': 275.49927920386904, 'shape': (1, 1, 1, 7, 15)}, 'tas_RCM3_gfdl_1981010103.nc': {'mu': 264.29543340773807, 'shape': (1, 1, 1, 7, 15)}, 'tas_CRCM_ncep_1986010103.nc': {'mu': 268.38143382352939, 'shape': (1, 1, 1, 8, 16)}, 'tas_CRCM_cgcm3_1986010103.nc': {'mu': 271.96783088235293, 'shape': (1, 1, 1, 8, 16)}, 'tas_CRCM_ccsm_1986010103.nc': {'mu': 262.36866191789215, 'shape': (1, 1, 1, 8, 16)}, 'tas_WRFG_cgcm3_1986010103.nc': {'mu': 274.17369962993422, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ccsm_1986010103.nc': {'mu': 260.47268194901318, 'shape': (1, 1, 1, 7, 14)}, 'tas_TMSL_ccsm_1981010103.nc': {'mu': 275.4296875, 'shape': (1, 1, 1, 7, 14)}, 'tas_WRFG_ccsm_1986010103.nc': {'mu': 260.7568359375, 'shape': (1, 1, 1, 7, 14)}, 'tas_RCM3_ncep_1986010103.nc': {'mu': 268.04431733630952, 'shape': (1, 1, 1, 7, 15)}, 'tas_CRCM_ncep_1981010103.nc': {'mu': 273.35757506127453, 'shape': (1, 1, 1, 8, 16)}, 'tas_ECP2_gfdl_1981010103.nc': {'mu': 261.00524662990193, 'shape': (1, 1, 1, 9, 17)}, 'tas_HRM3_ncep_1981010103.nc': {'mu': 274.20317263719511, 'shape': (1, 1, 1, 31, 22)}, 'tas_RCM3_cgcm3_1981010103.nc': {'mu': 270.27541387648807, 'shape': (1, 1, 1, 7, 15)}, 'tas_TMSL_gfdl_1981010100.nc': {'mu': 274.62939453125, 'shape': (1, 1, 1, 7, 15)}, 'tas_HRM3_gfdl_1981010103.nc': {'mu': 257.89143483231709, 'shape': (1, 1, 1, 31, 22)}, 'tas_WRFG_cgcm3_1981010103.nc': {'mu': 266.04286595394734, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ccsm_1981010103.nc': {'mu': 266.87697882401318, 'shape': (1, 1, 1, 7, 14)}, 'tas_WRFG_ncep_1981010103.nc': {'mu': 275.5278834292763, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ncep_1981010103.nc': {'mu': 273.7758275082237, 'shape': (1, 1, 1, 7, 14)}, 'tas_ECP2_gfdl_1986010103.nc': {'mu': 267.40525428921569, 'shape': (1, 1, 1, 9, 17)}, 'tas_MM5I_ncep_1986010103.nc': {'mu': 267.03638980263156, 'shape': (1, 1, 1, 7, 14)}, 'tas_RCM3_cgcm3_1986010103.nc': {'mu': 272.37116350446428, 'shape': (1, 1, 1, 7, 15)}, 'tas_WRFG_ncep_1986010103.nc': {'mu': 268.89250102796052, 'shape': (1, 1, 1, 7, 14)}, 'tas_HRM3_gfdl_1986010103.nc': {'mu': 270.4895674542683, 'shape': (1, 1, 1, 31, 22)}, 'tas_CRCM_ccsm_1981010103.nc': {'mu': 264.73590686274508, 'shape': (1, 1, 1, 8, 16)}}} for uri in os.listdir(data_dir): if uri.endswith('nc'): variable = uri.split('_')[0] for output_format in [ 'numpy', 'nc' ]: rd = RequestDataset(uri=uri,variable=variable)#,time_range=time_range) try: ops = OcgOperations(dataset=rd,geom='state_boundaries',select_ugid=[16],snippet=True, output_format=output_format) except DefinitionValidationError: ## rotated pole may not be written to netCDF crs = rd._get_crs_() if isinstance(crs,CFRotatedPole): continue else: raise try: ret = ops.execute() if output_format == 'numpy': ref = ret[16].values()[0].variables.values()[0].value mu = np.ma.mean(ref) r_mu = real[variable][uri]['mu'] try: self.assertAlmostEqual(r_mu,mu) except AssertionError: self.assertEqual(r_mu,np.ma.core.MaskedConstant) self.assertEqual(real[variable][uri]['shape'],ref.shape) if output_format == 'nc': with nc_scope(ret) as ds: try: grid_mapping = ds.variables[rd.variable].grid_mapping self.assertTrue(grid_mapping in ds.variables) except AttributeError: ## time slice files do not have a projection self.assertTrue('_TMSL_' in rd.uri) except ExtentError: if 'ECP2_ncep' in rd.uri: pass else: raise
def test_get_dimensioned_variables_two_variables_in_target_dataset(self): rd_orig = self.test_data.get_rd('cancm4_tas') dest_uri = os.path.join(self._test_dir, os.path.split(rd_orig.uri)[1]) shutil.copy2(rd_orig.uri, dest_uri) with nc_scope(dest_uri, 'a') as ds: var = ds.variables['tas'] outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions) outvar[:] = var[:] + 3 outvar.setncatts(var.__dict__) rd = RequestDataset(uri=dest_uri) self.assertEqual(rd.variable, ('tas', 'tasmax')) self.assertEqual(rd.variable, rd.alias)
def test_selecting_single_value(self): rd = self.test_data.get_rd('cancm4_tas') lat_index = 32 lon_index = 97 with nc_scope(rd.uri) as ds: lat_value = ds.variables['lat'][lat_index] lon_value = ds.variables['lon'][lon_index] data_values = ds.variables['tas'][:,lat_index,lon_index] ops = ocgis.OcgOperations(dataset=rd,geom=[lon_value,lat_value],search_radius_mult=0.1) ret = ops.execute() values = np.squeeze(ret[1]['tas'].variables['tas'].value) self.assertNumpyAll(data_values,values) geom = Point(lon_value,lat_value).buffer(0.001) ops = ocgis.OcgOperations(dataset=rd,geom=geom) ret = ops.execute() values = np.squeeze(ret[1]['tas'].variables['tas'].value) self.assertNumpyAll(data_values,values) geom = Point(lon_value-360.,lat_value).buffer(0.001) ops = ocgis.OcgOperations(dataset=rd,geom=geom) ret = ops.execute() values = np.squeeze(ret[1]['tas'].variables['tas'].value) self.assertNumpyAll(data_values,values) geom = Point(lon_value-360.,lat_value).buffer(0.001) ops = ocgis.OcgOperations(dataset=rd,geom=geom,aggregate=True,spatial_operation='clip') ret = ops.execute() values = np.squeeze(ret[1]['tas'].variables['tas'].value) self.assertNumpyAll(data_values,values) ops = ocgis.OcgOperations(dataset=rd,geom=[lon_value,lat_value], search_radius_mult=0.1,output_format='nc') ret = ops.execute() with nc_scope(ret) as ds: values = np.squeeze(ds.variables['tas'][:]) self.assertNumpyAll(data_values,values)
def test_calculate_opendap(self): ## test against an opendap target ensuring icclim and ocgis operations ## are equivalent in the netcdf output url = 'http://opendap.nmdc.eu/knmi/thredds/dodsC/IS-ENES/TESTSETS/tasmax_day_EC-EARTH_rcp26_r8i1p1_20760101-21001231.nc' calc_grouping = ['month'] rd = ocgis.RequestDataset(uri=url,variable='tasmax') calc_icclim = [{'func':'icclim_SU','name':'SU'}] ops = ocgis.OcgOperations(dataset=rd,calc=calc_icclim,calc_grouping=calc_grouping, output_format='nc',geom='state_boundaries',select_ugid=[10], prefix='icclim') ret_icclim = ops.execute() calc_ocgis = [{'func':'threshold','name':'SU','kwds':{'threshold':298.15,'operation':'gt'}}] ops = ocgis.OcgOperations(dataset=rd,calc=calc_ocgis,calc_grouping=calc_grouping, output_format='nc',geom='state_boundaries',select_ugid=[10], prefix='ocgis') ret_ocgis = ops.execute() ## variable and datasets will have different attributes, so adjust those ## before testing if the netCDFs are equal... with nc_scope(ret_icclim,'r') as ds_icclim: with nc_scope(ret_ocgis,'a') as ds_ocgis: ## strip the current attributes for key in ds_ocgis.ncattrs(): ds_ocgis.delncattr(key) for key in ds_ocgis.variables['SU'].ncattrs(): if not key.startswith('_'): ds_ocgis.variables['SU'].delncattr(key) ## make equivalent attributes for key in ds_icclim.ncattrs(): setattr(ds_ocgis,key,getattr(ds_icclim,key)) ## update the target variable attributes for key,value in ds_icclim.variables['SU'].__dict__.iteritems(): if not key.startswith('_'): setattr(ds_ocgis.variables['SU'],key,value) self.assertNcEqual(ret_icclim,ret_ocgis)
def test_with_overloads_real_data(self): ## copy the test file as the calendar attribute will be modified rd = self.test_data.get_rd('cancm4_tas') filename = os.path.split(rd.uri)[1] dest = os.path.join(self._test_dir, filename) shutil.copy2(rd.uri, dest) ## modify the calendar attribute with nc_scope(dest, 'a') as ds: self.assertEqual(ds.variables['time'].calendar, '365_day') ds.variables['time'].calendar = '365_days' ## assert the calendar is in fact changed on the source file with nc_scope(dest, 'r') as ds: self.assertEqual(ds.variables['time'].calendar, '365_days') rd2 = RequestDataset(uri=dest, variable='tas') field = rd2.get() ## the bad calendar will raise a value error when the datetimes are ## converted. with self.assertRaises(ValueError): field.temporal.value_datetime ## overload the calendar and confirm the datetime values are the same ## as the datetime values from the original good file rd3 = RequestDataset(uri=dest, variable='tas', t_calendar='365_day') field = rd3.get() self.assertNumpyAll(field.temporal.value_datetime, rd.get().temporal.value_datetime) ## pass as a dataset collection to operations and confirm the data may ## be written to a flat file. dates are converted in the process. time_range = (datetime.datetime(2001, 1, 1, 0, 0), datetime.datetime(2011, 1, 1, 0, 0)) dataset = [{'time_region': None, 'uri': dest, 'time_range': time_range, 'alias': u'tas', 't_units': u'days since 1850-1-1', 'variable': u'tas', 't_calendar': u'365_day'}] rdc = RequestDatasetCollection(dataset) ops = OcgOperations(dataset=rdc, geom='state_boundaries', select_ugid=[25], output_format='csv+') ops.execute()
def test_operations(self): uri = self.test_data.get_uri('cancm4_tas') rd = RequestDataset(uri=uri, variable='tas', # time_range=[datetime.datetime(2001,1,1),datetime.datetime(2003,12,31,23,59)] ) calc_grouping = ['month','year'] calc = [{'func':'dynamic_kernel_percentile_threshold','name':'tg10p','kwds':{'percentile':10,'width':5,'operation':'lt'}}] ops = OcgOperations(dataset=rd,calc_grouping=calc_grouping,calc=calc, output_format='nc') ret = ops.execute() with nc_scope(ret) as ds: ref = ds.variables['tg10p'][:] self.assertAlmostEqual(ref.mean(),2.9763946533203125)
def test_calculation_operations_to_nc(self): rd = self.test_data.get_rd('cancm4_tasmax_2011') slc = [None,None,None,[0,10],[0,10]] ops_ocgis = OcgOperations(calc=[{'func':'icclim_SU','name':'SU'}], calc_grouping=['month'], slice=slc, dataset=rd, output_format='nc') ret = ops_ocgis.execute() with nc_scope(ret) as ds: to_test = deepcopy(ds.__dict__) history = to_test.pop('history') self.assertEqual(history[111:187],' Calculation of SU indice (monthly climatology) from 2011-1-1 to 2020-12-31.') self.assertDictEqual(to_test,OrderedDict([(u'source_data_global_attributes', u'{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2010", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "64384802-3f0f-4ab4-b569-697bd5430854", "branch_time_YMDH": "2011:01:01:00", "CCCma_runid": "DHFP1B_E002_I2011_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2010", "frequency": "day", "creation_date": "2012-03-28T15:32:08Z", "history": "2012-03-28T15:32:08Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2010", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.8.0"}'), (u'title', u'ECA heat indice SU'), (u'references', u'ATBD of the ECA indices calculation (http://eca.knmi.nl/documents/atbd.pdf)'), (u'institution', u'Climate impact portal (http://climate4impact.eu)'), (u'comment', u' ')])) var = ds.variables['SU'] to_test = dict(var.__dict__) self.assertEqual(to_test,{'_FillValue':999999,u'units': u'days', u'standard_name': AbstractIcclimFunction.standard_name, u'long_name': 'Summer days (number of days where daily maximum temperature > 25 degrees)'})
def test_calculate_rotated_pole(self): tasmin_fake = self.test_data.get_rd('rotated_pole_ichec') tasmin_fake.alias = 'tasmin' tasmax_fake = deepcopy(tasmin_fake) tasmax_fake.alias = 'tasmax' rds = [tasmin_fake,tasmax_fake] for rd in rds: rd.time_region = {'year':[1973]} calc_ETR = [{'func':'icclim_ETR','name':'ETR','kwds':{'tasmin':'tasmin','tasmax':'tasmax'}}] ops = ocgis.OcgOperations(dataset=[tasmin_fake,tasmax_fake], calc=calc_ETR, calc_grouping=['year', 'month'], prefix = 'ETR_ocgis_icclim', output_format = 'nc', add_auxiliary_files=False) with nc_scope(ops.execute()) as ds: self.assertEqual(ds.variables['ETR'][:].shape,(12,103,106))
def test_standard_AbstractIcclimFunction(self): shapes = ([('month',), 12],[('month', 'year'), 24],[('year',),2]) ocgis.env.OVERWRITE = True keys = set(library_icclim._icclim_function_map.keys()) for klass in [ AbstractIcclimUnivariateSetFunction, AbstractIcclimMultivariateFunction]: for subclass in itersubclasses(klass): keys.remove(subclass.key) self.assertEqual([('month',),('month','year'),('year',)],subclass._allowed_temporal_groupings) for cg in CalcGrouping.iter_possible(): calc = [{'func':subclass.key,'name':subclass.key.split('_')[1]}] if klass == AbstractIcclimUnivariateSetFunction: rd = self.test_data.get_rd('cancm4_tas') rd.time_region = {'year':[2001,2002]} calc = [{'func':subclass.key,'name':subclass.key.split('_')[1]}] else: tasmin = self.test_data.get_rd('cancm4_tasmin_2001') tasmax = self.test_data.get_rd('cancm4_tasmax_2001') rd = [tasmin,tasmax] for r in rd: r.time_region = {'year':[2001,2002]} calc[0].update({'kwds':{'tasmin':'tasmin','tasmax':'tasmax'}}) try: ops = ocgis.OcgOperations(dataset=rd, output_format='nc', calc=calc, calc_grouping=cg, geom=[3.39,40.62,10.54,52.30]) ret = ops.execute() to_test = None for shape in shapes: if shape[0] == cg: to_test = shape[1] with nc_scope(ret) as ds: var = ds.variables[calc[0]['name']] self.assertEqual(var.dtype,subclass.dtype) self.assertEqual(var.shape,(to_test,5,4)) except DefinitionValidationError as e: if e.message.startswith('''OcgOperations validation raised an exception on the argument/operation "calc_grouping" with the message: The following temporal groupings are supported for ICCLIM: [('month',), ('month', 'year'), ('year',)]. The requested temporal group is:'''): pass else: raise(e) self.assertEqual(len(keys),0)
def test_calculation_operations_to_nc(self): rd = self.test_data.get_rd('cancm4_tas') slc = [None,None,None,[0,10],[0,10]] ops_ocgis = OcgOperations(calc=[{'func':'icclim_TG','name':'TG'}], calc_grouping=['month'], slice=slc, dataset=rd, output_format='nc') ret = ops_ocgis.execute() with nc_scope(ret) as ds: self.assertIn('Calculation of TG indice (monthly climatology)',ds.history) self.assertEqual(ds.title,'ECA temperature indice TG') var = ds.variables['TG'] ## check the JSON serialization self.assertEqual(ds.__dict__[AbstractIcclimFunction._global_attribute_source_name], u'{"institution": "CCCma (Canadian Centre for Climate Modelling and Analysis, Victoria, BC, Canada)", "institute_id": "CCCma", "experiment_id": "decadal2000", "source": "CanCM4 2010 atmosphere: CanAM4 (AGCM15i, T63L35) ocean: CanOM4 (OGCM4.0, 256x192L40) sea ice: CanSIM1 (Cavitating Fluid, T63 Gaussian Grid) land: CLASS2.7", "model_id": "CanCM4", "forcing": "GHG,Oz,SA,BC,OC,LU,Sl,Vl (GHG includes CO2,CH4,N2O,CFC11,effective CFC12)", "parent_experiment_id": "N/A", "parent_experiment_rip": "N/A", "branch_time": 0.0, "contact": "*****@*****.**", "references": "http://www.cccma.ec.gc.ca/models", "initialization_method": 1, "physics_version": 1, "tracking_id": "fac7bd83-dd7a-425b-b4dc-b5ab2e915939", "branch_time_YMDH": "2001:01:01:00", "CCCma_runid": "DHFP1B_E002_I2001_M01", "CCCma_parent_runid": "DHFP1_E002", "CCCma_data_licence": "1) GRANT OF LICENCE - The Government of Canada (Environment Canada) is the \\nowner of all intellectual property rights (including copyright) that may exist in this Data \\nproduct. You (as \\"The Licensee\\") are hereby granted a non-exclusive, non-assignable, \\nnon-transferable unrestricted licence to use this data product for any purpose including \\nthe right to share these data with others and to make value-added and derivative \\nproducts from it. This licence is not a sale of any or all of the owner\'s rights.\\n2) NO WARRANTY - This Data product is provided \\"as-is\\"; it has not been designed or \\nprepared to meet the Licensee\'s particular requirements. Environment Canada makes no \\nwarranty, either express or implied, including but not limited to, warranties of \\nmerchantability and fitness for a particular purpose. In no event will Environment Canada \\nbe liable for any indirect, special, consequential or other damages attributed to the \\nLicensee\'s use of the Data product.", "product": "output", "experiment": "10- or 30-year run initialized in year 2000", "frequency": "day", "creation_date": "2011-05-08T01:01:51Z", "history": "2011-05-08T01:01:51Z CMOR rewrote data to comply with CF standards and CMIP5 requirements.", "Conventions": "CF-1.4", "project_id": "CMIP5", "table_id": "Table day (28 March 2011) f9d6cfec5981bb8be1801b35a81002f0", "title": "CanCM4 model output prepared for CMIP5 10- or 30-year run initialized in year 2000", "parent_experiment": "N/A", "modeling_realm": "atmos", "realization": 2, "cmor_version": "2.5.4"}') ## load the original source attributes from the JSON string json.loads(ds.__dict__[AbstractIcclimFunction._global_attribute_source_name]) self.assertEqual(dict(var.__dict__),{'_FillValue':np.float32(1e20),u'units': u'K', u'standard_name': AbstractIcclimFunction.standard_name, u'long_name': u'Mean of daily mean temperature'})
def test_operations(self): uri = self.test_data.get_uri("cancm4_tas") rd = RequestDataset( uri=uri, variable="tas", # time_range=[datetime.datetime(2001,1,1),datetime.datetime(2003,12,31,23,59)] ) calc_grouping = ["month", "year"] calc = [ { "func": "dynamic_kernel_percentile_threshold", "name": "tg10p", "kwds": {"percentile": 10, "width": 5, "operation": "lt"}, } ] ops = OcgOperations(dataset=rd, calc_grouping=calc_grouping, calc=calc, output_format="nc") ret = ops.execute() with nc_scope(ret) as ds: ref = ds.variables["tg10p_tas"][:] self.assertAlmostEqual(ref.mean(), 2.9763946533203125)
def test_multiple_variables(self): conv_klasses = [CsvConverter, NcConverter] rd = self.test_data.get_rd('cancm4_tas') field = rd.get() var2 = deepcopy(field.variables['tas']) var2.alias = 'tas2' field.variables.add_variable(deepcopy(var2), assign_new_uid=True) field = field[:, 0:2, :, 0:5, 0:5] coll = self.get_spatial_collection(field=field) for conv_klass in conv_klasses: conv = conv_klass([coll], self._test_dir, 'ocgis_output_{0}'.format(conv_klass.__name__)) ret = conv.write() if conv_klass == CsvConverter: with open(ret, 'r') as f: reader = DictReader(f) aliases = set([row['ALIAS'] for row in reader]) self.assertEqual(set(['tas', 'tas2']), aliases) else: with nc_scope(ret) as ds: self.assertAlmostEqual(ds.variables['tas'][:].mean(), 247.08416015624999) self.assertNumpyAll(ds.variables['tas'][:], ds.variables['tas2'][:])
def test_init(self): with nc_scope(self.rd.uri, 'r') as ds: ncm = NcMetadata(ds) self.assertEqual(set(ncm.keys()), set(['dataset', 'variables', 'dimensions', 'file_format']))
def test_init_combinations(self): rd_orig = self.test_data.get_rd('cancm4_tas') dest_uri = os.path.join(self._test_dir, os.path.split(rd_orig.uri)[1]) shutil.copy2(rd_orig.uri, dest_uri) with nc_scope(dest_uri, 'a') as ds: var = ds.variables['tas'] outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions) outvar[:] = var[:] + 3 outvar.setncatts(var.__dict__) with nc_scope(dest_uri) as ds: self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys()))) keywords = dict( name=[None, 'foo'], uri=[None, dest_uri], variable=[None, 'tas', ['tas', 'tasmax'], 'crap'], alias=[None, 'tas', ['tas', 'tasmax'], ['tas_alias', 'tasmax_alias']], units=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy']], conform_units_to=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy'], [None, 'coulomb'], ['coulomb', 'coulomb']]) def itr_row(key, sequence): for element in sequence: yield ({key: element}) def itr_products_keywords(keywords): iterators = [itr_row(ki, vi) for ki, vi in keywords.iteritems()] for dictionaries in itertools.product(*iterators): yld = {} for dictionary in dictionaries: yld.update(dictionary) yield yld for k in itr_products_keywords(keywords): try: rd = RequestDataset(**k) self.assertEqual(rd._source_metadata, None) self.assertEqual(len(rd._variable), len(rd._units)) if k['name'] is None: self.assertEqual(rd.name, '_'.join(rd._alias)) else: self.assertEqual(rd.name, 'foo') for v in rd._variable: try: self.assertTrue(v in rd.source_metadata['variables'].keys()) except VariableNotFoundError: if 'crap' in rd._variable: self.assertEqual(rd._source_metadata, None) break if k['units'] is None and len(rd._variable) == 1: self.assertEqual(rd.units, None) self.assertEqual(rd._units, (None,)) try: field = rd.get() self.assertEqual(field.name, rd.name) self.assertEqual(set(field.variables.keys()), set(rd._alias)) except VariableNotFoundError: if 'crap' in rd._variable: continue else: raise except RequestValidationError: if 'coulomb' in get_tuple(k['conform_units_to']): continue else: raise except RequestValidationError as e: ## uris cannot be None if k['uri'] is None: pass ## variables cannot be None elif k['variable'] is None: pass ## 'crap' is not a real variable name elif k['conform_units_to'] is not None and (k['conform_units_to'] == 'crap' or \ 'crap' in k['conform_units_to']): pass ## conform_units_to must match units element-wise elif k['conform_units_to'] is not None and k['variable'] is not None and \ len(k['conform_units_to']) != len(k['variable']): pass ## aliases must occur for each variable elif len(get_tuple(k['alias'])) != len(get_tuple(k['variable'])): pass ## units must occur for each variable elif len(get_tuple(k['units'])) != len(get_tuple(k['variable'])): pass ## bad unit definition ## 'crap' is not a real variable name elif k['units'] is not None and (k['units'] == 'crap' or \ 'crap' in k['units']): pass ## alway need a uri and variable elif k['uri'] is None: pass else: print k raise except: print k raise