def ncstation2cov(save_coverage=True):
# Open the netcdf dataset
ds = Dataset('test_data/usgs.nc')
# Itemize the variable names that we want to include in the coverage
var_names = ['time','lat','lon','z','streamflow','water_temperature',]
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a ParameterContext object for each of the variables in the dataset and add them to the ParameterDictionaryl
for v in var_names:
var = ds.variables[v]
pcontext = ParameterContext(v, param_type=QuantityType(var.dtype.char))
if 'units' in var.ncattrs():
pcontext.uom = var.getncattr('units')
if 'long_name' in var.ncattrs():
pcontext.description = var.getncattr('long_name')
if '_FillValue' in var.ncattrs():
pcontext.fill_value = var.getncattr('_FillValue')
# Set the reference_frame for the coordinate parameters
if v == 'time':
pcontext.reference_frame = AxisTypeEnum.TIME
elif v == 'lat':
pcontext.reference_frame = AxisTypeEnum.LAT
elif v == 'lon':
pcontext.reference_frame = AxisTypeEnum.LON
elif v == 'z':
pcontext.reference_frame = AxisTypeEnum.HEIGHT
pdict.add_context(pcontext)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS.standard_temporal()
scrs = CRS.lat_lon_height()
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('sample station coverage_model', pdict, sdom, tdom)
# Insert the timesteps (automatically expands other arrays)
tvar=ds.variables['time']
scov.insert_timesteps(tvar.size)
# Add data to the parameters - NOT using setters at this point, direct assignment to arrays
for v in var_names:
var = ds.variables[v]
var.set_auto_maskandscale(False)
# TODO: Sort out how to leave the coordinates sparse internally and only broadcast during read
scov.range_value[v][:] = var[:]
if save_coverage:
SimplexCoverage.save(scov, 'test_data/usgs.cov')
return scov
def oneparamcov(save_coverage=False, in_memory=False):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory)
# Insert some timesteps (automatically expands other arrays)
scov.insert_timesteps(10)
# Add data for the parameter
scov.set_parameter_values('time', value=np.arange(10))
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/sample.cov')
return scov
def test_plot_1():
from coverage_model.test.examples import SimplexCoverage
import matplotlib.pyplot as plt
cov=SimplexCoverage.load('test_data/usgs.cov')
log.debug('Plot the \'water_temperature\' and \'streamflow\' for all times')
wtemp = cov.get_parameter_values('water_temperature')
wtemp_pc = cov.get_parameter_context('water_temperature')
sflow = cov.get_parameter_values('streamflow')
sflow_pc = cov.get_parameter_context('streamflow')
times = cov.get_parameter_values('time')
time_pc = cov.get_parameter_context('time')
fig = plt.figure()
ax1 = fig.add_subplot(2,1,1)
ax1.plot(times,wtemp)
ax1.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom))
ax1.set_ylabel('{0} ({1})'.format(wtemp_pc.name, wtemp_pc.uom))
ax2 = fig.add_subplot(2,1,2)
ax2.plot(times,sflow)
ax2.set_xlabel('{0} ({1})'.format(time_pc.name, time_pc.uom))
ax2.set_ylabel('{0} ({1})'.format(sflow_pc.name, sflow_pc.uom))
plt.show(0)
def methodized_read():
from coverage_model.test.examples import SimplexCoverage
import numpy as np
import os
log.debug('============ Station ============')
pth = 'test_data/usgs.cov'
if not os.path.exists(pth):
raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth))
cov=SimplexCoverage.load(pth)
ra=np.zeros([0])
log.debug('\n>> All data for first timestep\n')
log.debug(cov.get_parameter_values('water_temperature',0,None,ra))
log.debug('\n>> All data\n')
log.debug(cov.get_parameter_values('water_temperature',None,None,None))
log.debug('\n>> All data for second, fifth and sixth timesteps\n')
log.debug(cov.get_parameter_values('water_temperature',[[1,4,5]],None,None))
log.debug('\n>> First datapoint (in x) for every 5th timestep\n')
log.debug(cov.get_parameter_values('water_temperature',slice(0,None,5),0,None))
log.debug('\n>> First datapoint for first 10 timesteps, passing DOA objects\n')
tdoa = DomainOfApplication(slice(0,10))
sdoa = DomainOfApplication(0)
log.debug(cov.get_parameter_values('water_temperature',tdoa,sdoa,None))
log.debug('\n============ Grid ============')
pth = 'test_data/ncom.cov'
if not os.path.exists(pth):
raise SystemError('Cannot proceed, \'{0}\' file must exist. Run the \'ncstation2cov()\' function to generate the file.'.format(pth))
cov=SimplexCoverage.load(pth)
ra=np.zeros([0])
log.debug('\n>> All data for first timestep\n')
log.debug(cov.get_parameter_values('water_temp',0,None,ra))
log.debug('\n>> All data\n')
log.debug(cov.get_parameter_values('water_temp',None,None,None))
log.debug('\n>> All data for first, fourth, and fifth timesteps\n')
log.debug(cov.get_parameter_values('water_temp',[[0,3,4]],None,None))
log.debug('\n>> Data from z=0, y=10, x=10 for every 2nd timestep\n')
log.debug(cov.get_parameter_values('water_temp',slice(0,None,2),[0,10,10],None))
log.debug('\n>> Data from z=0-10, y=10, x=10 for the first 2 timesteps, passing DOA objects\n')
tdoa = DomainOfApplication(slice(0,2))
sdoa = DomainOfApplication([slice(0,10),10,10])
log.debug(cov.get_parameter_values('water_temp',tdoa,sdoa,None))
def ncgrid2cov(save_coverage=True):
# Open the netcdf dataset
ds = Dataset('test_data/ncom.nc')
# Itemize the variable names that we want to include in the coverage
var_names = ['time','lat','lon','depth','water_u','water_v','salinity','water_temp',]
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a ParameterContext object for each of the variables in the dataset and add them to the ParameterDictionary
for v in var_names:
var = ds.variables[v]
pcontext = ParameterContext(v, param_type=QuantityType(value_encoding=ds.variables[v].dtype.char))
if 'units' in var.ncattrs():
pcontext.uom = var.getncattr('units')
if 'long_name' in var.ncattrs():
pcontext.description = var.getncattr('long_name')
if '_FillValue' in var.ncattrs():
pcontext.fill_value = var.getncattr('_FillValue')
# Set the reference_frame for the coordinate parameters
if v == 'time':
pcontext.reference_frame = AxisTypeEnum.TIME
elif v == 'lat':
pcontext.reference_frame = AxisTypeEnum.LAT
elif v == 'lon':
pcontext.reference_frame = AxisTypeEnum.LON
elif v == 'depth':
pcontext.reference_frame = AxisTypeEnum.HEIGHT
pdict.add_context(pcontext)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT, AxisTypeEnum.HEIGHT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal'), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [34,57,89]), scrs, MutabilityEnum.IMMUTABLE) # 3d spatial topology (grid)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('sample grid coverage_model', pdict, sdom, tdom)
# Insert the timesteps (automatically expands other arrays)
tvar=ds.variables['time']
scov.insert_timesteps(tvar.size)
# Add data to the parameters - NOT using setters at this point, direct assignment to arrays
for v in var_names:
var = ds.variables[v]
var.set_auto_maskandscale(False)
arr = var[:]
# TODO: Sort out how to leave these sparse internally and only broadcast during read
if v == 'depth':
z,_,_ = my_meshgrid(arr,np.zeros([57]),np.zeros([89]),indexing='ij',sparse=True)
scov.range_value[v][:] = z
elif v == 'lat':
_,y,_ = my_meshgrid(np.zeros([34]),arr,np.zeros([89]),indexing='ij',sparse=True)
scov.range_value[v][:] = y
elif v == 'lon':
_,_,x = my_meshgrid(np.zeros([34]),np.zeros([57]),arr,indexing='ij',sparse=True)
scov.range_value[v][:] = x
else:
scov.range_value[v][:] = var[:]
if save_coverage:
SimplexCoverage.save(scov, 'test_data/ncom.cov')
return scov
def samplecov(save_coverage=True):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('sample coverage_model', pdict, sdom, tdom)
# Insert some timesteps (automatically expands other arrays)
scov.insert_timesteps(10)
# Add data for each parameter
scov.set_parameter_values('time', value=np.arange(10))
scov.set_parameter_values('lat', value=45)
scov.set_parameter_values('lon', value=-71)
# make a random sample of 10 values between 23 and 26
# Ref: http://docs.scipy.org/doc/numpy/reference/generated/numpy.random.random_sample.html#numpy.random.random_sample
# --> To sample multiply the output of random_sample by (b-a) and add a
tvals=np.random.random_sample(10)*(26-23)+23
scov.set_parameter_values('temp', value=tvals)
scov.set_parameter_values('conductivity', value=np.random.random_sample(10)*(30-20)+20)
if save_coverage:
SimplexCoverage.save(scov, 'test_data/sample.cov')
return scov
def nospatialcov(save_coverage=False, in_memory=False):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL)
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32')))
quant_ctxt.long_name = 'example of a parameter of type QuantityType'
quant_ctxt.uom = 'degree_Celsius'
pdict.add_context(quant_ctxt)
const_ctxt = ParameterContext('constant', param_type=ConstantType())
const_ctxt.long_name = 'example of a parameter of type ConstantType'
pdict.add_context(const_ctxt)
arr_ctxt = ParameterContext('array', param_type=ArrayType())
arr_ctxt.long_name = 'example of a parameter of type ArrayType with base_type ndarray (resolves to \'object\')'
pdict.add_context(arr_ctxt)
arr2_ctxt = ParameterContext('array2', param_type=ArrayType())
arr2_ctxt.long_name = 'example of a parameter of type ArrayType with base_type object'
pdict.add_context(arr2_ctxt)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, temporal_domain=tdom, in_memory_storage=in_memory)
# Insert some timesteps (automatically expands other arrays)
scov.insert_timesteps(10)
# Add data for each parameter
scov.set_parameter_values('quantity_time', value=np.arange(10))
scov.set_parameter_values('quantity', value=np.random.random_sample(10)*(26-23)+23)
scov.set_parameter_values('constant', value=20)
arrval = []
arr2val = []
for x in xrange(scov.num_timesteps): # One value (which IS an array) for each member of the domain
arrval.append(np.random.bytes(np.random.randint(1,20)))
arr2val.append(np.random.random_sample(np.random.randint(1,10)))
scov.set_parameter_values('array', value=arrval)
scov.set_parameter_values('array2', value=arr2val)
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov')
return scov
def ptypescov(save_coverage=False, in_memory=False):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
quant_t_ctxt = ParameterContext('quantity_time', param_type=QuantityType(value_encoding=np.dtype('int64')), variability=VariabilityEnum.TEMPORAL)
quant_t_ctxt.reference_frame = AxisTypeEnum.TIME
quant_t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(quant_t_ctxt)
cnst_int_ctxt = ParameterContext('const_int', param_type=ConstantType(QuantityType(value_encoding=np.dtype('int32'))), variability=VariabilityEnum.NONE)
cnst_int_ctxt.long_name = 'example of a parameter of type ConstantType, base_type int32'
cnst_int_ctxt.reference_frame = AxisTypeEnum.LAT
cnst_int_ctxt.uom = 'degree_north'
pdict.add_context(cnst_int_ctxt)
cnst_flt_ctxt = ParameterContext('const_float', param_type=ConstantType(), variability=VariabilityEnum.NONE)
cnst_flt_ctxt.long_name = 'example of a parameter of type QuantityType, base_type float (default)'
cnst_flt_ctxt.reference_frame = AxisTypeEnum.LON
cnst_flt_ctxt.uom = 'degree_east'
pdict.add_context(cnst_flt_ctxt)
# func_ctxt = ParameterContext('function', param_type=FunctionType(QuantityType(value_encoding=np.dtype('float32'))))
# func_ctxt.long_name = 'example of a parameter of type FunctionType'
# pdict.add_context(func_ctxt)
quant_ctxt = ParameterContext('quantity', param_type=QuantityType(value_encoding=np.dtype('float32')))
quant_ctxt.long_name = 'example of a parameter of type QuantityType'
quant_ctxt.uom = 'degree_Celsius'
pdict.add_context(quant_ctxt)
arr_ctxt = ParameterContext('array', param_type=ArrayType())
arr_ctxt.long_name = 'example of a parameter of type ArrayType, will be filled with variable-length \'byte-string\' data'
pdict.add_context(arr_ctxt)
rec_ctxt = ParameterContext('record', param_type=RecordType())
rec_ctxt.long_name = 'example of a parameter of type RecordType, will be filled with dictionaries'
pdict.add_context(rec_ctxt)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', pdict, tdom, sdom, in_memory)
# Insert some timesteps (automatically expands other arrays)
scov.insert_timesteps(10)
# Add data for each parameter
scov.set_parameter_values('quantity_time', value=np.arange(10))
scov.set_parameter_values('const_int', value=45.32) # Set a constant directly, with incorrect data type (fixed under the hood)
scov.set_parameter_values('const_float', value=make_range_expr(-71.11)) # Set with a properly formed constant expression
scov.set_parameter_values('quantity', value=np.random.random_sample(10)*(26-23)+23)
# # Setting three range expressions such that indices 0-2 == 10, 3-7 == 15 and >=8 == 20
# scov.set_parameter_values('function', value=make_range_expr(10, 0, 3, min_incl=True, max_incl=False, else_val=-999.9))
# scov.set_parameter_values('function', value=make_range_expr(15, 3, 8, min_incl=True, max_incl=False, else_val=-999.9))
# scov.set_parameter_values('function', value=make_range_expr(20, 8, min_incl=True, max_incl=False, else_val=-999.9))
arrval = []
recval = []
letts='abcdefghij'
for x in xrange(scov.num_timesteps):
arrval.append(np.random.bytes(np.random.randint(1,20))) # One value (which is a byte string) for each member of the domain
d = {letts[x]: letts[x:]}
recval.append(d) # One value (which is a dict) for each member of the domain
scov.set_parameter_values('array', value=arrval)
scov.set_parameter_values('record', value=recval)
if in_memory and save_coverage:
SimplexCoverage.pickle_save(scov, 'test_data/ptypes.cov')
return scov