def test_alias(self):
path = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(uri=path)
self.assertIsNone(rd._alias)
self.assertEqual(rd.alias, 'tas')
rd.alias = 'temperature'
self.assertEqual(rd.alias, 'temperature')
def __init__(self,uri=None,variable=None,interface_overload={},request_dataset=None):
self.request_dataset = request_dataset
if self.request_dataset is None:
self.uri = uri
self.variable = variable
self.alias = None
self.did = None
if self.variable is None:
try:
self.ds = None
rootgrp = nc.Dataset(uri)
self.meta = NcMetadata(rootgrp)
finally:
rootgrp.close()
else:
from ocgis.api.request.base import RequestDataset
kwds = {'uri':uri,'variable':variable}
kwds.update(interface_overload)
rd = RequestDataset(**kwds)
self.ds = rd.get()
self.meta = self.ds.meta
else:
self.uri = self.request_dataset.uri
self.variable = self.request_dataset.variable
self.ds = self.request_dataset.get()
self.meta = self.ds.meta
self.alias = self.request_dataset.alias
self.did = self.request_dataset.did
def test_inspect_as_dct(self):
variables = [
self.variable,
None,
'foo',
'time'
]
for variable in variables:
try:
rd = RequestDataset(self.uri, variable)
ret = rd.inspect_as_dct()
except VariableNotFoundError:
if variable == 'foo':
continue
else:
raise
except ValueError:
if variable == 'time':
continue
else:
raise
ref = ret['derived']
self.assertEqual(ref['End Date'], '2021-01-01 00:00:00')
self.assertEqual(ref.keys(),
['Name', 'Count', 'Has Bounds', 'Data Type', 'Start Date', 'End Date', 'Calendar', 'Units',
'Resolution (Days)', 'Spatial Reference', 'Proj4 String', 'Extent', 'Geometry Interface',
'Resolution'])
def test_dev_get_split_shapefile(self):
raise SkipTest('development only')
self.set_debug()
shp_path = '/home/benkoziol/l/data/nfie/linked_catchment_shapefiles/linked_13-RioGrande.shp'
rd = RequestDataset(uri=shp_path)
field = rd.get()
self.log.debug('loading from file')
field.geom.value
node_count = map(get_node_count, field.geom.value)
select = np.array(node_count) > 10000
to_split = field['GRIDCODE'][select]
for gc in to_split.value.flat:
self.log.debug('target gridcode: {}'.format(gc))
idx = np.where(field['GRIDCODE'].value == gc)[0][0]
target_geom = field.geom.value[idx]
split_geom = get_split_polygon_by_node_threshold(target_geom, 10000)
# write_fiona(split_geom, gc)
self.assertAlmostEqual(split_geom.area, target_geom.area)
field.geom.value[idx] = split_geom
self.assertAlmostEqual(field.geom.value[idx].area, target_geom.area)
self.log.debug(field.geom.geom_type)
# field.geom[select].parent.write('/tmp/rio-grande-assembled.shp', driver=DriverVector)
# write_fiona(field.geom.value, 'rio-grande-assembled')
self.log.debug('writing shapefile')
field.write('/tmp/rio-grande-assembled.shp', driver=DriverVector)
def test_operations_two_steps(self):
## get the request dataset to use as the basis for the percentiles
uri = self.test_data.get_uri('cancm4_tas')
variable = 'tas'
rd = RequestDataset(uri=uri,variable=variable)
## this is the underly OCGIS dataset object
nc_basis = rd.get()
## NOTE: if you want to subset the basis by time, this step is necessary
# nc_basis = nc_basis.get_between('temporal',datetime.datetime(2001,1,1),datetime.datetime(2003,12,31,23,59))
## these are the values to use when calculating the percentile basis. it
## may be good to wrap this in a function to have memory freed after the
## percentile structure array is computed.
all_values = nc_basis.variables[variable].value
## these are the datetime objects used for window creation
temporal = nc_basis.temporal.value_datetime
## additional parameters for calculating the basis
percentile = 10
width = 5
## get the structure array
from ocgis.calc.library.index.dynamic_kernel_percentile import DynamicDailyKernelPercentileThreshold
daily_percentile = DynamicDailyKernelPercentileThreshold.get_daily_percentile(all_values,temporal,percentile,width)
## perform the calculation using the precomputed basis. in this case,
## the basis and target datasets are the same, so the RequestDataset is
## reused.
calc_grouping = ['month','year']
kwds = {'percentile':percentile,'width':width,'operation':'lt','daily_percentile':daily_percentile}
calc = [{'func':'dynamic_kernel_percentile_threshold','name':'tg10p','kwds':kwds}]
ops = OcgOperations(dataset=rd,calc_grouping=calc_grouping,calc=calc,
output_format='nc')
ret = ops.execute()
## if we want to return the values as a three-dimenional numpy array the
## method below will do this. note the interface arrangement for the next
## release will alter this slightly.
ops = OcgOperations(dataset=rd,calc_grouping=calc_grouping,calc=calc,
output_format='numpy')
arrs = ops.execute()
## reference the returned numpy data. the first key is the geometry identifier.
## 1 in this case as this is the default for no selection geometry. the second
## key is the request dataset alias and the third is the calculation name.
## the variable name is appended to the end of the calculation to maintain
## a unique identifier.
tg10p = arrs[1]['tas'].variables['tg10p'].value
## if we want the date information for the temporal groups date attributes
date_parts = arrs[1]['tas'].temporal.date_parts
assert(date_parts.shape[0] == tg10p.shape[1])
## these are the representative datetime objects
rep_dt = arrs[1]['tas'].temporal.value_datetime
## and these are the lower and upper time bounds on the date groups
bin_bounds = arrs[1]['tas'].temporal.bounds_datetime
## confirm we have values for each month and year (12*10)
ret_ds = nc.Dataset(ret)
try:
self.assertEqual(ret_ds.variables['tg10p'].shape,(120,64,128))
finally:
ret_ds.close()
def test_inspect_as_dct(self):
variables = [
self.variable,
None,
'foo',
'time'
]
for variable in variables:
try:
rd = RequestDataset(self.uri,variable)
ret = rd.inspect_as_dct()
except KeyError:
if variable == 'foo':
continue
else:
raise
except ValueError:
if variable == 'time':
continue
else:
raise
except AssertionError:
if variable is not None:
raise
else:
continue
ref = ret['derived']
if variable is None:
self.assertEqual(ref,{'End Date': '2020-12-31 12:00:00', 'Start Date': '2011-01-01 12:00:00'})
else:
self.assertEqual(ref['End Date'],'2021-01-01 00:00:00')
def test_name(self):
path = ShpCabinet().get_shp_path('state_boundaries')
rd = RequestDataset(uri=path, driver='vector')
self.assertIsNotNone(rd.name)
rd = RequestDataset(uri=path, driver='vector', name='states')
self.assertEqual(rd.name, 'states')
field = rd.get()
self.assertEqual(field.name, 'states')
def test_read_write_projections(self):
data_dir = '/usr/local/climate_data/narccap'
ocgis.env.DIR_DATA = data_dir
ocgis.env.OVERWRITE = True
real = {'pr': {'pr_RCM3_cgcm3_1986010103.nc': {'mu': 2.7800052478033606e-07, 'shape': (1, 1, 1, 7, 15)}, 'pr_MM5I_ncep_1981010103.nc': {'mu': 3.3648159627007675e-08, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_ncep_1986010103.nc': {'mu': 9.7176247154926553e-09, 'shape': (1, 1, 1, 7, 15)}, 'pr_CRCM_ncep_1986010103.nc': {'mu': 1.1799650910219663e-26, 'shape': (1, 1, 1, 8, 16)}, 'pr_CRCM_cgcm3_1981010103.nc': {'mu': 2.6299784818262446e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_MM5I_ncep_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_HRM3_ncep_1981010103.nc': {'mu': 5.507401147596971e-10, 'shape': (1, 1, 1, 31, 22)}, 'pr_RCM3_cgcm3_1981010103.nc': {'mu': 1.18896825283411e-05, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_gfdl_1986010100.nc': {'mu': 2.0890602963450161e-07, 'shape': (1, 1, 1, 7, 15)}, 'pr_WRFG_cgcm3_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_ECP2_gfdl_1981010103.nc': {'mu': 6.1180394635919263e-06, 'shape': (1, 1, 1, 9, 17)}, 'pr_CRCM_ncep_1981010103.nc': {'mu': 2.767125774613198e-05, 'shape': (1, 1, 1, 8, 16)}, 'pr_HRM3_gfdl_1986010103.nc': {'mu': 4.1377767579766305e-06, 'shape': (1, 1, 1, 31, 22)}, 'pr_RCM3_gfdl_1981010103.nc': {'mu': -5.1954553518551086e-24, 'shape': (1, 1, 1, 7, 15)}, 'pr_HRM3_ncep_1986010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 31, 22)}, 'pr_TMSL_ccsm_1986010103.nc': {'mu': 3.734873736402074e-07, 'shape': (1, 1, 1, 7, 14)}, 'pr_HRM3_gfdl_1981010103.nc': {'mu': 5.2488248024374339e-07, 'shape': (1, 1, 1, 31, 22)}, 'pr_WRFG_ccsm_1986010103.nc': {'mu': 0.00010390303979970907, 'shape': (1, 1, 1, 7, 14)}, 'pr_MM5I_ccsm_1986010103.nc': {'mu': 5.0342728890858494e-07, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_ccsm_1981010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_cgcm3_1981010103.nc': {'mu': 0.0, 'shape': (1, 1, 1, 7, 14)}, 'pr_WRFG_ncep_1981010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_ncep_1981010103.nc': {'mu': 7.637150009118376e-06, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_ccsm_1981010103.nc': {'mu': 9.641077844117023e-27, 'shape': (1, 1, 1, 7, 14)}, 'pr_RCM3_gfdl_1986010103.nc': {'mu': 1.0929620614097941e-05, 'shape': (1, 1, 1, 7, 15)}, 'pr_TMSL_gfdl_1981010100.nc': {'mu': 1.3174895956811014e-10, 'shape': (1, 1, 1, 7, 15)}, 'pr_CRCM_ccsm_1981010103.nc': {'mu': 1.6264247653238914e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_WRFG_ncep_1986010103.nc': {'mu': np.ma.core.MaskedConstant, 'shape': (1, 1, 1, 7, 14)}, 'pr_CRCM_cgcm3_1986010103.nc': {'mu': 3.152432755621917e-06, 'shape': (1, 1, 1, 8, 16)}, 'pr_MM5I_ccsm_1981010103.nc': {'mu': 1.5723979779044096e-09, 'shape': (1, 1, 1, 7, 14)}, 'pr_CRCM_ccsm_1986010103.nc': {'mu': 1.1736681164406678e-05, 'shape': (1, 1, 1, 8, 16)}, 'pr_ECP2_gfdl_1986010103.nc': {'mu': 9.865492043614972e-06, 'shape': (1, 1, 1, 9, 17)}}, 'tas': {'tas_TMSL_gfdl_1986010100.nc': {'mu': 272.1787109375, 'shape': (1, 1, 1, 7, 15)}, 'tas_RCM3_gfdl_1986010103.nc': {'mu': 257.74983723958331, 'shape': (1, 1, 1, 7, 15)}, 'tas_HRM3_ncep_1986010103.nc': {'mu': 272.10732660060978, 'shape': (1, 1, 1, 31, 22)}, 'tas_WRFG_ccsm_1981010103.nc': {'mu': 259.1943359375, 'shape': (1, 1, 1, 7, 14)}, 'tas_TMSL_ccsm_1986010103.nc': {'mu': 271.766502490942, 'shape': (1, 1, 1, 7, 14)}, 'tas_CRCM_cgcm3_1981010103.nc': {'mu': 256.05007276348039, 'shape': (1, 1, 1, 8, 16)}, 'tas_RCM3_ncep_1981010103.nc': {'mu': 275.49927920386904, 'shape': (1, 1, 1, 7, 15)}, 'tas_RCM3_gfdl_1981010103.nc': {'mu': 264.29543340773807, 'shape': (1, 1, 1, 7, 15)}, 'tas_CRCM_ncep_1986010103.nc': {'mu': 268.38143382352939, 'shape': (1, 1, 1, 8, 16)}, 'tas_CRCM_cgcm3_1986010103.nc': {'mu': 271.96783088235293, 'shape': (1, 1, 1, 8, 16)}, 'tas_CRCM_ccsm_1986010103.nc': {'mu': 262.36866191789215, 'shape': (1, 1, 1, 8, 16)}, 'tas_WRFG_cgcm3_1986010103.nc': {'mu': 274.17369962993422, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ccsm_1986010103.nc': {'mu': 260.47268194901318, 'shape': (1, 1, 1, 7, 14)}, 'tas_TMSL_ccsm_1981010103.nc': {'mu': 275.4296875, 'shape': (1, 1, 1, 7, 14)}, 'tas_WRFG_ccsm_1986010103.nc': {'mu': 260.7568359375, 'shape': (1, 1, 1, 7, 14)}, 'tas_RCM3_ncep_1986010103.nc': {'mu': 268.04431733630952, 'shape': (1, 1, 1, 7, 15)}, 'tas_CRCM_ncep_1981010103.nc': {'mu': 273.35757506127453, 'shape': (1, 1, 1, 8, 16)}, 'tas_ECP2_gfdl_1981010103.nc': {'mu': 261.00524662990193, 'shape': (1, 1, 1, 9, 17)}, 'tas_HRM3_ncep_1981010103.nc': {'mu': 274.20317263719511, 'shape': (1, 1, 1, 31, 22)}, 'tas_RCM3_cgcm3_1981010103.nc': {'mu': 270.27541387648807, 'shape': (1, 1, 1, 7, 15)}, 'tas_TMSL_gfdl_1981010100.nc': {'mu': 274.62939453125, 'shape': (1, 1, 1, 7, 15)}, 'tas_HRM3_gfdl_1981010103.nc': {'mu': 257.89143483231709, 'shape': (1, 1, 1, 31, 22)}, 'tas_WRFG_cgcm3_1981010103.nc': {'mu': 266.04286595394734, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ccsm_1981010103.nc': {'mu': 266.87697882401318, 'shape': (1, 1, 1, 7, 14)}, 'tas_WRFG_ncep_1981010103.nc': {'mu': 275.5278834292763, 'shape': (1, 1, 1, 7, 14)}, 'tas_MM5I_ncep_1981010103.nc': {'mu': 273.7758275082237, 'shape': (1, 1, 1, 7, 14)}, 'tas_ECP2_gfdl_1986010103.nc': {'mu': 267.40525428921569, 'shape': (1, 1, 1, 9, 17)}, 'tas_MM5I_ncep_1986010103.nc': {'mu': 267.03638980263156, 'shape': (1, 1, 1, 7, 14)}, 'tas_RCM3_cgcm3_1986010103.nc': {'mu': 272.37116350446428, 'shape': (1, 1, 1, 7, 15)}, 'tas_WRFG_ncep_1986010103.nc': {'mu': 268.89250102796052, 'shape': (1, 1, 1, 7, 14)}, 'tas_HRM3_gfdl_1986010103.nc': {'mu': 270.4895674542683, 'shape': (1, 1, 1, 31, 22)}, 'tas_CRCM_ccsm_1981010103.nc': {'mu': 264.73590686274508, 'shape': (1, 1, 1, 8, 16)}}}
for uri in os.listdir(data_dir):
if uri.endswith('nc'):
variable = uri.split('_')[0]
for output_format in [
'numpy',
'nc'
]:
rd = RequestDataset(uri=uri,variable=variable)#,time_range=time_range)
try:
ops = OcgOperations(dataset=rd,geom='state_boundaries',select_ugid=[16],snippet=True,
output_format=output_format)
except DefinitionValidationError:
## rotated pole may not be written to netCDF
crs = rd._get_crs_()
if isinstance(crs,CFRotatedPole):
continue
else:
raise
try:
ret = ops.execute()
if output_format == 'numpy':
ref = ret[16].values()[0].variables.values()[0].value
mu = np.ma.mean(ref)
r_mu = real[variable][uri]['mu']
try:
self.assertAlmostEqual(r_mu,mu)
except AssertionError:
self.assertEqual(r_mu,np.ma.core.MaskedConstant)
self.assertEqual(real[variable][uri]['shape'],ref.shape)
if output_format == 'nc':
with nc_scope(ret) as ds:
try:
grid_mapping = ds.variables[rd.variable].grid_mapping
self.assertTrue(grid_mapping in ds.variables)
except AttributeError:
## time slice files do not have a projection
self.assertTrue('_TMSL_' in rd.uri)
except ExtentError:
if 'ECP2_ncep' in rd.uri:
pass
else:
raise
def test_init_driver(self):
uri = ShpCabinet().get_shp_path('state_boundaries')
rd = RequestDataset(uri=uri, driver='vector')
self.assertIsNotNone(rd.variable)
self.assertIsInstance(rd.get(), Field)
uri_nc = self.test_data.get_uri('cancm4_tas')
rd = RequestDataset(uri_nc)
self.assertIsInstance(rd.driver, DriverNetcdf)
rd = RequestDataset(uri_nc, driver='vector')
with self.assertRaises(ValueError):
assert rd.variable
def __init__(self, uri=None, variable=None, request_dataset=None, meta=None):
if meta is None and uri is None and request_dataset is None:
raise ValueError('At least one of "uri", "request_dataset", or "meta" must be provided.')
self.uri = uri
self.variable = variable
self.meta = meta
self.request_dataset = request_dataset
if self.request_dataset is None and self.meta is None:
if self.uri is not None and self.variable is not None:
from ocgis.api.request.base import RequestDataset
self.request_dataset = RequestDataset(uri=self.uri, variable=self.variable)
self.alias = None
self.did = None
self.ds = None
if self.request_dataset is None:
if self.meta is None:
rootgrp = nc.Dataset(self.uri)
try:
self.meta = NcMetadata(rootgrp)
finally:
rootgrp.close()
else:
self.uri = self.request_dataset.uri
self.variable = self.request_dataset.variable
self.ds = self.request_dataset.get()
self.meta = self.ds.meta
self.alias = self.request_dataset.alias
self.did = self.request_dataset.did
def test_keyword_time_subset_func(self):
def _func_(value, bounds=None):
indices = []
for ii, v in enumerate(value.flat):
if v.month == 6:
indices.append(ii)
return indices
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd, time_subset_func=_func_, geom='state_boundaries', geom_select_uid=[20])
ret = ops.execute()
for v in ret[20]['tas'].temporal.value_datetime:
self.assertEqual(v.month, 6)
rd = self.test_data.get_rd('cancm4_tas')
ops = OcgOperations(dataset=rd, time_subset_func=_func_, geom='state_boundaries', geom_select_uid=[20],
output_format=constants.OUTPUT_FORMAT_NETCDF)
ret = ops.execute()
rd_out = RequestDataset(ret)
for v in rd_out.get().temporal.value_datetime:
self.assertEqual(v.month, 6)
def test_with_overloads_real_data(self):
# copy the test file as the calendar attribute will be modified
rd = self.test_data.get_rd('cancm4_tas')
filename = os.path.split(rd.uri)[1]
dest = os.path.join(self.current_dir_output, filename)
shutil.copy2(rd.uri, dest)
# modify the calendar attribute
with nc_scope(dest, 'a') as ds:
self.assertEqual(ds.variables['time'].calendar, '365_day')
ds.variables['time'].calendar = '365_days'
# assert the calendar is in fact changed on the source file
with nc_scope(dest, 'r') as ds:
self.assertEqual(ds.variables['time'].calendar, '365_days')
rd2 = RequestDataset(uri=dest, variable='tas')
field = rd2.get()
# the bad calendar will raise a value error when the datetimes are converted.
with self.assertRaises(ValueError):
field.temporal.value_datetime
# overload the calendar and confirm the datetime values are the same as the datetime values from the original
# good file
rd3 = RequestDataset(uri=dest, variable='tas', t_calendar='365_day')
field = rd3.get()
self.assertNumpyAll(field.temporal.value_datetime, rd.get().temporal.value_datetime)
# pass as a dataset collection to operations and confirm the data may be written to a flat file. dates are
# converted in the process.
time_range = (datetime.datetime(2001, 1, 1, 0, 0), datetime.datetime(2011, 1, 1, 0, 0))
dataset = [{'time_region': None,
'uri': dest,
'time_range': time_range,
'alias': u'tas',
't_units': u'days since 1850-1-1',
'variable': u'tas',
't_calendar': u'365_day'}]
rdc = RequestDatasetCollection(dataset)
ops = OcgOperations(dataset=rdc, geom='state_boundaries', select_ugid=[25],
output_format=constants.OUTPUT_FORMAT_SHAPEFILE)
ops.execute()
def test_inspect(self):
rd = RequestDataset(self.uri, self.variable)
with self.print_scope() as ps:
rd.inspect()
self.assertTrue(len(ps.storage) >= 1)
uri = ShpCabinet().get_shp_path('state_boundaries')
rd = RequestDataset(uri=uri, driver='vector')
with self.print_scope() as ps:
rd.inspect()
self.assertTrue(len(ps.storage) >= 1)
# test with a request dataset having no dimensioned variables
path = self.get_temporary_file_path('bad.nc')
with self.nc_scope(path, 'w') as ds:
ds.createDimension('foo')
var = ds.createVariable('foovar', int, dimensions=('foo',))
var.a_name = 'a name'
rd = RequestDataset(uri=path)
with self.print_scope() as ps:
rd.inspect()
self.assertTrue(len(ps.storage) >= 1)
def test_alias_change_after_init_two_variables(self):
kwds = self.get_multiple_variable_request_dataset_dictionary()
rd = RequestDataset(**kwds)
self.assertEqual(rd.name, 'tas_tasmax')
with self.assertRaises(RequestValidationError):
rd.alias = 'foo'
rd.alias = ['foo', 'foo2']
self.assertEqual(rd.alias, ('foo', 'foo2'))
self.assertEqual(rd.name, 'foo_foo2')
with self.assertRaises(RequestValidationError):
rd.units = 'crap'
with self.assertRaises(RequestValidationError):
rd.units = ('crap', 'crap')
rd.units = ['celsius', 'celsius']
self.assertEqual(rd.units, ('celsius', 'celsius'))
def test_get_autodiscovered_driver(self):
uri_shp = '/path/to/shapefile.shp'
uri_nc = '/path/to/netcdf/file/foo.nc'
uri_nc_opendap = 'http://cida.usgs.gov/thredds/dodsC/maurer/maurer_brekke_w_meta.ncml'
driver = RequestDataset._get_autodiscovered_driver_(uri_shp)
self.assertEqual(driver, DriverVector)
for poss in [uri_nc, [uri_nc, uri_nc], uri_nc_opendap, [uri_nc_opendap, uri_nc_opendap]]:
driver = RequestDataset._get_autodiscovered_driver_(poss)
self.assertEqual(driver, DriverNetcdf)
with self.assertRaises(RequestValidationError):
RequestDataset._get_autodiscovered_driver_('something/meaninglyess.foobuar')
with self.assertRaises(RequestValidationError):
RequestDataset._get_autodiscovered_driver_('something/meaninglyess')
def test_get_field_nonequivalent_units_in_source_data(self):
new_path = self.test_data.copy_file('cancm4_tas', self.current_dir_output)
# put non-equivalent units on the source data and attempto to conform
with nc_scope(new_path, 'a') as ds:
ds.variables['tas'].units = 'coulomb'
rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius')
with self.assertRaises(RequestValidationError):
rd.get()
# remove units altogether
with nc_scope(new_path, 'a') as ds:
ds.variables['tas'].delncattr('units')
rd = RequestDataset(uri=new_path, variable='tas', conform_units_to='celsius')
with self.assertRaises(NoUnitsError):
rd.get()
def test_inspect_method(self):
rd = RequestDataset(self.uri, self.variable)
rd.inspect()
def test_init_combinations(self):
rd_orig = self.test_data.get_rd('cancm4_tas')
dest_uri = os.path.join(self.current_dir_output, os.path.split(rd_orig.uri)[1])
shutil.copy2(rd_orig.uri, dest_uri)
with nc_scope(dest_uri, 'a') as ds:
var = ds.variables['tas']
outvar = ds.createVariable(var._name + 'max', var.dtype, var.dimensions)
outvar[:] = var[:] + 3
outvar.setncatts(var.__dict__)
with nc_scope(dest_uri) as ds:
self.assertTrue(set(['tas', 'tasmax']).issubset(set(ds.variables.keys())))
keywords = dict(
name=[None, 'foo'],
uri=[None, dest_uri],
variable=[None, 'tas', ['tas', 'tasmax'], 'crap'],
alias=[None, 'tas', ['tas', 'tasmax'], ['tas_alias', 'tasmax_alias']],
units=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'], ['crap', 'crappy']],
conform_units_to=[None, [None, None], ['celsius', 'fahrenheit'], 'crap', [None, 'kelvin'],
['crap', 'crappy'], [None, 'coulomb'], ['coulomb', 'coulomb']])
def itr_row(key, sequence):
for element in sequence:
yield ({key: element})
def itr_products_keywords(keywords):
iterators = [itr_row(ki, vi) for ki, vi in keywords.iteritems()]
for dictionaries in itertools.product(*iterators):
yld = {}
for dictionary in dictionaries:
yld.update(dictionary)
yield yld
for k in itr_products_keywords(keywords):
try:
rd = RequestDataset(**k)
self.assertEqual(rd._source_metadata, None)
self.assertEqual(len(get_tuple(rd.variable)), len(get_tuple(rd.units)))
if k['name'] is None:
self.assertEqual(rd.name, '_'.join(get_tuple(rd.alias)))
else:
self.assertEqual(rd.name, 'foo')
for v in rd._variable:
try:
self.assertTrue(v in rd.source_metadata['variables'].keys())
except VariableNotFoundError:
if 'crap' in rd._variable:
self.assertEqual(rd._source_metadata, None)
break
if k['units'] is None and len(rd._variable) == 1:
self.assertEqual(rd.units, None)
self.assertEqual(rd._units, None)
try:
field = rd.get()
self.assertEqual(field.name, rd.name)
self.assertEqual(set(field.variables.keys()), set(get_tuple(rd.alias)))
except VariableNotFoundError:
if 'crap' in rd._variable:
continue
else:
raise
except RequestValidationError:
if 'coulomb' in get_tuple(k['conform_units_to']):
continue
else:
raise
except RequestValidationError as e:
# uris cannot be None
if k['uri'] is None:
pass
# variables cannot be None
elif k['variable'] is None:
pass
# 'crap' is not a real variable name
elif k['conform_units_to'] is not None and (k['conform_units_to'] == 'crap' or \
'crap' in k['conform_units_to']):
pass
# conform_units_to must match units element-wise
elif k['conform_units_to'] is not None and k['variable'] is not None and \
len(k['conform_units_to']) != len(k['variable']):
pass
# aliases must occur for each variable
elif len(get_tuple(k['alias'])) != len(get_tuple(k['variable'])):
pass
# units must occur for each variable
elif len(get_tuple(k['units'])) != len(get_tuple(k['variable'])):
pass
# bad unit definition
# 'crap' is not a real variable name
elif k['units'] is not None and (k['units'] == 'crap' or \
'crap' in k['units']):
pass
# alway need a uri and variable
elif k['uri'] is None:
pass
else:
raise
except:
raise
def test_init_variable_not_found(self):
rd = self.test_data.get_rd('cancm4_tas')
rd_bad = RequestDataset(uri=rd.uri, variable='crap')
with self.assertRaises(VariableNotFoundError):
rd_bad.get()
class Inspect(object):
"""
Inspect a local or remote dataset returning a printout similar to `ncdump`_.
>>> from ocgis import Inspect
...
>>> # Just do an dataset attribute dump.
>>> ip = Inspect('/my/local/dataset')
>>> print(ip)
...
>>> # Get variable-specific info.
>>> ip = Inspect('/my/local/dataset',variable='tas')
>>> print(ip)
:param uri: Absolute path to data's location.
:type uri: str
:param variable: Specific variable to inspect.
:type variable: str
:param interface_overload: Overloads for autodiscover.
:type interface_overload: dict
:param meta: Use this metadata object in place of the one created internally.
:type meta: :class:`ocgis.NcMetadata`
.. _ncdump: http://www.unidata.ucar.edu/software/netcdf/docs/netcdf/ncdump.html
"""
def __init__(self, uri=None, variable=None, request_dataset=None, meta=None):
if meta is None and uri is None and request_dataset is None:
raise ValueError('At least one of "uri", "request_dataset", or "meta" must be provided.')
self.uri = uri
self.variable = variable
self.meta = meta
self.request_dataset = request_dataset
if self.request_dataset is None and self.meta is None:
if self.uri is not None and self.variable is not None:
from ocgis.api.request.base import RequestDataset
self.request_dataset = RequestDataset(uri=self.uri, variable=self.variable)
self.alias = None
self.did = None
self.ds = None
if self.request_dataset is None:
if self.meta is None:
rootgrp = nc.Dataset(self.uri)
try:
self.meta = NcMetadata(rootgrp)
finally:
rootgrp.close()
else:
self.uri = self.request_dataset.uri
self.variable = self.request_dataset.variable
self.ds = self.request_dataset.get()
self.meta = self.ds.meta
self.alias = self.request_dataset.alias
self.did = self.request_dataset.did
def __repr__(self):
msg = ''
if self.request_dataset is None:
lines = self.get_report_no_variable()
else:
lines = self.get_report()
for line in lines:
msg += line + '\n'
return msg
@property
def _t(self):
return self.ds.temporal
@property
def _s(self):
return self.ds.spatial
@property
def _l(self):
return self.ds.level
def get_temporal_report(self):
try:
if self._t.format_time:
res = int(self._t.resolution)
try:
start_date, end_date = self._t.extent_datetime
# # the times may not be formattable
except ValueError as e:
if e.message == 'year is out of range':
start_date, end_date = self._t.extent
else:
ocgis_lh(exc=e, logger='inspect')
else:
res = 'NA (non-formatted times requested)'
start_date, end_date = self._t.extent
# # raised if the temporal dimension has a single value. possible with
## snippet or a small dataset...
except ResolutionError:
res, start_date, end_date = ['NA (singleton)'] * 3
n = len(self._t.value)
# # if the calendar attribute is not set, the feature should not be masked
calendar = self.meta['variables'][self._t.name]['attrs'].get('calendar')
if calendar is None:
calendar = 'None (will assume "standard")'
units = self._t.units
lines = []
lines.append(' Start Date = {0}'.format(start_date))
lines.append(' End Date = {0}'.format(end_date))
lines.append(' Calendar = {0}'.format(calendar))
lines.append(' Units = {0}'.format(units))
lines.append('Resolution (Days) = {0}'.format(res))
lines.append(' Count = {0}'.format(n))
## append information on bounds
if self._t.bounds is not None:
has_bounds = True
else:
has_bounds = False
lines.append(' Has Bounds = {0}'.format(has_bounds))
return lines
def get_spatial_report(self):
res = self._s.grid.resolution
extent = self._s.grid.extent
itype = self._s.geom.get_highest_order_abstraction().__class__.__name__
projection = self.ds.spatial.crs
lines = []
lines.append('Spatial Reference = {0}'.format(projection.__class__.__name__))
lines.append(' Proj4 String = {0}'.format(projection.sr.ExportToProj4()))
lines.append(' Extent = {0}'.format(extent))
lines.append(' Interface Type = {0}'.format(itype))
lines.append(' Resolution = {0}'.format(res))
lines.append(' Count = {0}'.format(self._s.grid.uid.reshape(-1).shape[0]))
return lines
def get_level_report(self):
if self._l is None:
lines = ['No level dimension found.']
else:
lines = []
lines.append('Level Variable = {0}'.format(self._l.name))
lines.append(' Count = {0}'.format(self._l.value.shape[0]))
# # append information on bounds
if self._l.bounds is not None:
has_bounds = True
else:
has_bounds = False
lines.append(' Has Bounds = {0}'.format(has_bounds))
return lines
def get_dump_report(self):
return (self.meta._get_lines_())
def get_report_no_variable(self):
lines = ['', 'URI = {0}'.format(self.uri)]
lines.append('VARIABLE = {0}'.format(self.variable))
lines.append('')
lines += self.get_dump_report()
return lines
def get_report(self):
# # a variable target is required for this method
if self.variable is None:
raise (AttributeError('A "variable" target is required.'))
mp = [
{'=== Temporal =============': self.get_temporal_report},
{'=== Spatial ==============': self.get_spatial_report},
{'=== Level ================': self.get_level_report},
{'=== Dump =================': self.get_dump_report}
]
lines = ['', 'URI = {0}'.format(self.uri)]
lines.append('VARIABLE = {0}'.format(self.variable))
lines.append('ALIAS = {0}'.format(self.alias))
lines.append('DID = {0}'.format(self.did))
lines.append('')
for dct in mp:
for key, value in dct.iteritems():
lines.append(key)
lines.append('')
for line in value():
lines.append(line)
lines.append('')
return lines
def _as_dct_(self):
ret = self.meta.copy()
# # without a target variable, attempt to set start and end dates.
if self.variable is None:
ds = nc.Dataset(self.uri, 'r')
try:
time = ds.variables['time']
time_bounds = [time[0], time[-1]]
time_bounds = nc.num2date(time_bounds, time.units, calendar=time.calendar)
derived = {'Start Date': str(time_bounds[0]), 'End Date': str(time_bounds[1])}
except:
warn('Time variable not found or improperly attributed. Setting "derived" key to None.')
derived = None
finally:
ds.close()
## we can get derived values
else:
derived = OrderedDict()
to_add = self.get_temporal_report() + self.get_spatial_report() + self.get_level_report()
for row in to_add:
try:
key, value = re.split(' = ', row, maxsplit=1)
## here to catch oddities of the returns
except ValueError:
if row == 'No level dimension found.':
continue
else:
raise
key = key.strip()
derived.update({key: value})
ret.update({'derived': derived})
return ret
