def test_load(self):
rd = self.test_data.get_rd('cancm4_tas')
gi = NcDataset(request_dataset=rd)
spatial = gi.spatial
self.assertTrue(spatial.is_360)
self.assertEqual(spatial.grid.shape, (64, 128))
self.assertTrue(isinstance(spatial.projection, WGS84))
level = gi.level
self.assertEqual(level, None)
temporal = gi.temporal
self.assertEqual(
str(temporal.extent),
'(datetime.datetime(2001, 1, 1, 0, 0), datetime.datetime(2011, 1, 1, 0, 0))'
)
res = temporal.resolution
self.assertAlmostEqual(int(res), 1)
trng = [
datetime.datetime(2001, 1, 1),
datetime.datetime(2001, 12, 31, 23, 59)
]
new_ds = gi.get_subset(temporal=trng)
self.assertEqual(new_ds.temporal.shape[0], 365)
self.assertEqual(new_ds.value.shape, (365, 1, 64, 128))
def test_aggregate(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
sd = ShpDataset('state_boundaries')
sd.spatial.unwrap_geoms()
utah = sd[23].spatial.geom[0]
ods = ods.get_subset(spatial_operation='clip', igeom=utah)
ods.aggregate()
self.assertEqual(ods.value.shape, (3650, 1, 1, 1))
self.assertEqual(ods.raw_value.shape, (3650, 1, 3, 3))
self.assertNotEqual(ods.spatial.vector.raw_weights.shape,
ods.spatial.vector.weights.shape)
self.assertEqual(ods.spatial.vector.uid[0], 1)
def test_aggregate(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
sd = ShpDataset('state_boundaries')
sd.spatial.unwrap_geoms()
utah = sd[23].spatial.geom[0]
ods = ods.get_subset(spatial_operation='clip',igeom=utah)
ods.aggregate()
self.assertEqual(ods.value.shape,(3650,1,1,1))
self.assertEqual(ods.raw_value.shape,(3650,1,3,3))
self.assertNotEqual(ods.spatial.vector.raw_weights.shape,
ods.spatial.vector.weights.shape)
self.assertEqual(ods.spatial.vector.uid[0],1)
def test_masking(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
nods = ods[0:31,0:5,0:5]
geoms = nods.spatial.vector.geom
igeom = [geoms[0,2]] + list(geoms[1,:])
igeom = MultiPolygon(igeom)
self.assertFalse(np.any(nods.value.mask))
sods = ods.get_subset(spatial_operation='intersects',igeom=igeom)
geom_mask = sods.spatial.vector.geom.mask
value_mask = sods.value.mask[0,:,:,:]
self.assertTrue(np.all(geom_mask == value_mask))
def test_masking(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
nods = ods[0:31, 0:5, 0:5]
geoms = nods.spatial.vector.geom
igeom = [geoms[0, 2]] + list(geoms[1, :])
igeom = MultiPolygon(igeom)
self.assertFalse(np.any(nods.value.mask))
sods = ods.get_subset(spatial_operation='intersects', igeom=igeom)
geom_mask = sods.spatial.vector.geom.mask
value_mask = sods.value.mask[0, :, :, :]
self.assertTrue(np.all(geom_mask == value_mask))
def generate_dataset_report(folder):
'''
Generate a descriptive report on the data included in the use case.
:param folder: Path to the folder containing the NARCCAP example data.
:type folder: str
:returns: Path to the report file.
:rtype: str
'''
ocgis.env.DIR_DATA = folder
rds = parse_narccap_filenames(folder)
rdc = ocgis.RequestDatasetCollection(rds)
headers = ['DID', 'Filenames', 'Variable', 'Time Start', 'Time End']
(fd, name) = tempfile.mkstemp(suffix='.csv')
f = open(name, 'w')
writer = csv.writer(f)
writer.writerow(headers)
for rd in rdc:
logging.info(rd)
ds = NcDataset(rd)
to_write = [
rd.did, [os.path.split(uri)[1] for uri in rd.uri], rd.variable,
ds.temporal.value[0], ds.temporal.value[-1]
]
writer.writerow(to_write)
f.close()
return (name)
def __init__(self,
uri=None,
variable=None,
interface_overload={},
request_dataset=None):
self.request_dataset = request_dataset
if self.request_dataset is None:
self.uri = uri
self.variable = variable
self.alias = None
self.did = None
if self.variable is None:
try:
self.ds = None
rootgrp = nc.Dataset(uri)
self.meta = NcMetadata(rootgrp)
finally:
rootgrp.close()
else:
from ocgis.api.request import RequestDataset
kwds = {'uri': uri, 'variable': variable}
kwds.update(interface_overload)
rd = RequestDataset(**kwds)
self.ds = NcDataset(request_dataset=rd)
self.meta = self.ds.metadata
else:
self.uri = self.request_dataset.uri
self.variable = self.request_dataset.variable
self.ds = self.request_dataset.ds
self.meta = self.request_dataset.ds.metadata
self.alias = self.request_dataset.alias
self.did = self.request_dataset.did
def compute(dataset,
calc,
calc_grouping,
tile_dimension,
verbose=False,
prefix=None):
tile_dimension = int(tile_dimension)
if tile_dimension <= 0:
raise (ValueError('"tile_dimension" must be greater than 0'))
orig_oc = ocgis.env.OPTIMIZE_FOR_CALC
ocgis.env.OPTIMIZE_FOR_CALC = True
try:
ods = NcDataset(request_dataset=dataset)
shp = ods.spatial.grid.shape
if verbose: print('getting schema...')
schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension)
if verbose: print('getting fill file...')
fill_file = ocgis.OcgOperations(dataset=dataset,
file_only=True,
calc=calc,
calc_grouping=calc_grouping,
output_format='nc',
prefix=prefix).execute()
if verbose: print('output file is: {0}'.format(fill_file))
if verbose:
lschema = len(schema)
print('tile count: {0}'.format(lschema))
fds = nc.Dataset(fill_file, 'a')
if verbose:
progress = ProgressBar('tiles progress')
for ctr, indices in enumerate(schema.itervalues(), start=1):
row = indices['row']
col = indices['col']
ret = ocgis.OcgOperations(dataset=dataset,
slice=[None, row, col],
calc=calc,
calc_grouping=calc_grouping).execute()
for variable in ret[1].variables.iterkeys():
ref = ret[1].calc[variable]
for k, v in ref.iteritems():
vref = fds.variables[k]
if len(vref.shape) == 3:
vref[:, row[0]:row[1], col[0]:col[1]] = v
elif len(vref.shape) == 4:
vref[:, :, row[0]:row[1], col[0]:col[1]] = v
else:
raise (NotImplementedError)
fds.sync()
if verbose:
progress.progress(int((float(ctr) / lschema) * 100))
fds.close()
finally:
ocgis.env.OPTIMIZE_FOR_CALC = orig_oc
if verbose:
progress.endProgress()
print('complete.')
return (fill_file)
def test_temporal_group(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
grouping = ['month']
ods.temporal.set_grouping(grouping)
self.assertEqual(ods.temporal.group.value.shape[0], 12)
for row in ods.temporal.group.get_iter():
pass
def test_abstraction_point(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd,abstraction='point')
with self.assertRaises(AttributeError):
ods.spatial.shape
self.assertTrue(isinstance(ods.spatial.vector,NcPointDimension))
geom = ods.spatial.vector.geom
self.assertEqual(geom.shape,(64,128))
self.assertTrue(isinstance(geom[0,0],Point))
weights = ods.spatial.vector.weights
self.assertEqual(weights.max(),1.)
poly = make_poly((-62,59),(87,244))
nods = ods.get_subset(spatial_operation='intersects',igeom=poly)
with self.assertRaises(AttributeError):
nods.spatial.shape
self.assertNotEqual(ods.spatial.vector.shape,nods.spatial.vector.shape)
nods.aggregate()
self.assertEqual(nods.spatial.vector.geom.shape,(1,1))
self.assertTrue(isinstance(nods.spatial.vector.geom[0,0],Point))
def test_initial_subset(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
ods._load_slice.update({'T': slice(0, 1)})
tp = ods.temporal
to_check = [tp.value, tp.uid, tp.real_idx]
for tc in to_check:
self.assertEqual(tc.shape, (1, ))
self.assertEqual(tp.bounds.shape, (1, 2))
self.assertEqual(ods.value.shape, (1, 1, 64, 128))
def test_load(self):
rd = self.test_data.get_rd('cancm4_tas')
gi = NcDataset(request_dataset=rd)
spatial = gi.spatial
self.assertTrue(spatial.is_360)
self.assertEqual(spatial.grid.shape,(64,128))
self.assertTrue(isinstance(spatial.projection,WGS84))
level = gi.level
self.assertEqual(level,None)
temporal = gi.temporal
self.assertEqual(str(temporal.extent),'(datetime.datetime(2001, 1, 1, 0, 0), datetime.datetime(2011, 1, 1, 0, 0))')
res = temporal.resolution
self.assertAlmostEqual(int(res),1)
trng = [datetime.datetime(2001,1,1),datetime.datetime(2001,12,31,23,59)]
new_ds = gi.get_subset(temporal=trng)
self.assertEqual(new_ds.temporal.shape[0],365)
self.assertEqual(new_ds.value.shape,(365,1,64,128))
def test_abstraction_point(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd, abstraction='point')
with self.assertRaises(AttributeError):
ods.spatial.shape
self.assertTrue(isinstance(ods.spatial.vector, NcPointDimension))
geom = ods.spatial.vector.geom
self.assertEqual(geom.shape, (64, 128))
self.assertTrue(isinstance(geom[0, 0], Point))
weights = ods.spatial.vector.weights
self.assertEqual(weights.max(), 1.)
poly = make_poly((-62, 59), (87, 244))
nods = ods.get_subset(spatial_operation='intersects', igeom=poly)
with self.assertRaises(AttributeError):
nods.spatial.shape
self.assertNotEqual(ods.spatial.vector.shape,
nods.spatial.vector.shape)
nods.aggregate()
self.assertEqual(nods.spatial.vector.geom.shape, (1, 1))
self.assertTrue(isinstance(nods.spatial.vector.geom[0, 0], Point))
def test_slice(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
sods = ods[:, 0:5, 0:5]
self.assertEqual(sods.value.shape, (3650, 1, 5, 5))
self.assertEqual(sods.spatial.vector.geom.shape, (5, 5))
sods = ods[0, 0:5, 0:5]
self.assertEqual(sods.value.shape, (1, 1, 5, 5))
sods = ods[0, 0, 0]
self.assertEqual(sods.value.shape, (1, 1, 1, 1))
grouping = ['month']
ods.temporal.set_grouping(grouping)
sods = ods[0:31, 0:5, 0:5]
self.assertEqual(sods.temporal.group, None)
sods.temporal.set_grouping(grouping)
self.assertEqual(sods.temporal.group.value.shape[0], 1)
def test_project(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
dest = WGS84()
with self.assertRaises(NotImplementedError):
ods.project(dest)
def test_get_iter(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)[0:31,0:5,0:5]
itr = ods.get_iter_value()
self.assertEqual(len(list(itr)),31*5*5)
def test_project(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)
dest = WGS84()
with self.assertRaises(NotImplementedError):
ods.project(dest)
def test_get_iter(self):
rd = self.test_data.get_rd('cancm4_tas')
ods = NcDataset(request_dataset=rd)[0:31, 0:5, 0:5]
itr = ods.get_iter_value()
self.assertEqual(len(list(itr)), 31 * 5 * 5)
def compute(dataset,
calc,
calc_grouping,
tile_dimension,
verbose=False,
prefix=None):
'''
:type dataset: RequestDatasetCollection
'''
assert (isinstance(dataset, RequestDatasetCollection))
assert (type(calc) in (list, tuple))
tile_dimension = int(tile_dimension)
if tile_dimension <= 0:
raise (ValueError('"tile_dimension" must be greater than 0'))
orig_oc = ocgis.env.OPTIMIZE_FOR_CALC
ocgis.env.OPTIMIZE_FOR_CALC = False
try:
## load some data into the optimize store
print('loading into optimize store...')
for rd in dataset:
if verbose: print('request dataset', rd.alias)
ocgis.env._optimize_store[rd.alias] = {}
ocgis.env._optimize_store[
rd.alias]['_value_datetime'] = rd.ds.temporal.value_datetime
ocgis.env._optimize_store[
rd.alias]['_bounds_datetime'] = rd.ds.temporal.bounds_datetime
if calc_grouping is not None:
rd.ds.temporal.set_grouping(calc_grouping)
ocgis.env._optimize_store[
rd.alias]['group'] = rd.ds.temporal.group
rd._ds = None
## tell the software we are optimizing for calculations
ocgis.env.OPTIMIZE_FOR_CALC = True
ods = NcDataset(request_dataset=dataset[0])
shp = ods.spatial.grid.shape
if verbose: print('getting schema...')
schema = tile.get_tile_schema(shp[0], shp[1], tile_dimension)
if verbose: print('getting fill file...')
fill_file = ocgis.OcgOperations(dataset=dataset,
file_only=True,
calc=calc,
calc_grouping=calc_grouping,
output_format='nc',
prefix=prefix).execute()
if verbose: print('output file is: {0}'.format(fill_file))
if verbose:
lschema = len(schema)
print('tile count: {0}'.format(lschema))
fds = nc.Dataset(fill_file, 'a')
if verbose:
progress = ProgressBar('tiles progress')
for ctr, indices in enumerate(schema.itervalues(), start=1):
row = indices['row']
col = indices['col']
ret = ocgis.OcgOperations(dataset=dataset,
slice=[None, row, col],
calc=calc,
calc_grouping=calc_grouping).execute()
for vref, v in iter_variable_values(ret[1], fds):
if len(vref.shape) == 3:
vref[:, row[0]:row[1], col[0]:col[1]] = v
elif len(vref.shape) == 4:
vref[:, :, row[0]:row[1], col[0]:col[1]] = v
else:
raise (NotImplementedError(vref.shape))
fds.sync()
if verbose:
progress.progress(int((float(ctr) / lschema) * 100))
fds.close()
finally:
ocgis.env.OPTIMIZE_FOR_CALC = orig_oc
ocgis.env._optimize_store = {}
if verbose:
progress.endProgress()
print('complete.')
return (fill_file)