def gridtype_example():
"""Create a simple grid."""
example = GridType("example")
example["a"] = BaseType("a", data=np.arange(30*50).reshape(30, 50))
example["x"] = BaseType("x", data=np.arange(30))
example["y"] = BaseType("y", data=np.arange(50))
return example
def setUp(self):
sz = 12
time = numpy.arange(float(0), float(sz))
data = numpy.arange(float(sz))
for ind in range(sz):
data[ind] = numpy.random.random()
ds = DatasetType(name='SimpleGridData')
g = GridType(name='TimeSeries')
# The name in the dictionary must match the name in the basetype
g['timeseries'] = BaseType(name='timeseries',
data=data,
shape=data.shape,
type=Float32,
dimensions=('time'))
g['time'] = BaseType(name='time',
data=time,
shape=(sz, ),
type=Float32)
ds[g.name] = g
self.ds1 = ds
self.tc = timeseries_consumer.TimeseriesConsumer()
yield self.tc.plc_init()
def mean(dataset, var, axis=0):
"""Calculate the mean of an array along a given axis.
The input variable should be either a ``GridType`` or ``BaseType``. The
function will return an object of the same type with the mean applied.
"""
if not isinstance(var, (GridType, BaseType)):
raise ConstraintExpressionError(
'Function "mean" should be used on an array or grid.')
axis = int(axis)
dims = tuple(dim for i, dim in enumerate(var.dimensions) if i != axis)
# process basetype
if isinstance(var, BaseType):
return BaseType(name=var.name,
data=np.mean(var.data[:], axis=axis),
dimensions=dims,
attributes=var.attributes)
# process grid
out = GridType(name=var.name, attributes=var.attributes)
out[var.array.name] = BaseType(name=var.array.name,
data=np.mean(var.array.data[:], axis=axis),
dimensions=dims,
attributes=var.array.attributes)
for dim in dims:
out[dim] = BaseType(name=dim,
data=var[dim].data[:],
dimensions=(dim, ),
attributes=var[dim].attributes)
return out
def stream(self):
sz = 10
time = numpy.arange(float(self.index), float(self.index + sz))
self.index += sz
data = numpy.arange(float(sz))
for ind in range(sz):
data[ind] = numpy.random.random()
ds = DatasetType(name='SimpleGridData')
g = GridType(name='Time Series')
# The name in the dictionary must match the name in the basetype
g['timeseries'] = BaseType(name='timeseries',
data=data,
shape=data.shape,
type=Float32,
dimensions=('time'))
g['time'] = BaseType(name='time',
data=time,
shape=(sz, ),
type=Float32)
ds[g.name] = g
msg = dap_tools.ds2dap_msg(ds)
yield self.send(self.deliver, 'data', msg.encode())
def simple_grid_dataset():
"""
@brief Create a simple dap grid dataset
Just use the pydap interface - passing dicts does not make sense here.
"""
# Convert metadata and data to a dap dataset
ds = DatasetType(name='SimpleGridData')
g = GridType(name='grid')
data = numpy.arange(24.)
data.shape = (4, 2, 3)
# The name in the dictionary must match the name in the basetype
g['a'] = BaseType(name='a',
data=data,
shape=data.shape,
type=Float32,
dimensions=('time', 'x', 'y'))
g['time'] = BaseType(name='time',
data=numpy.arange(4.),
shape=(4, ),
type=Float64)
g['x'] = BaseType(name='x',
data=numpy.arange(2.),
shape=(2, ),
type=Float64)
g['y'] = BaseType(name='y',
data=numpy.arange(3.),
shape=(3, ),
type=Float64)
ds[g.name] = g
return ds
def test_StructureType_set_data():
"""Test that data is propagated to children."""
var = StructureType("var", value=42, one="1")
var["one"] = BaseType("one")
var["two"] = BaseType("two")
var.data = [10, 20]
assert (var["one"].data == 10)
assert (var["two"].data == 20)
def test_set_data(self):
"""Test that data is propagated to children."""
var = StructureType("var", value=42, one="1")
var["one"] = BaseType("one")
var["two"] = BaseType("two")
var.data = [10, 20]
self.assertEqual(var["one"].data, 10)
self.assertEqual(var["two"].data, 20)
def setUp(self):
"""Create a simple grid."""
example = GridType("example")
example["a"] = BaseType("a", data=np.arange(30 * 50).reshape(30, 50))
example["x"] = BaseType("x", data=np.arange(30))
example["y"] = BaseType("y", data=np.arange(50))
self.example = example
def test_StructureType_repr():
"""Test ``__repr__`` method."""
var = StructureType("var")
assert (repr(var) == "<StructureType with children >")
var["one"] = BaseType("one")
var["two"] = BaseType("two")
assert (repr(var) == "<StructureType with children 'one', 'two'>")
def test_repr(self):
"""Test ``__repr__`` method."""
var = StructureType("var")
self.assertEqual(repr(var), "<StructureType with children >")
var["one"] = BaseType("one")
var["two"] = BaseType("two")
self.assertEqual(repr(var),
"<StructureType with children 'one', 'two'>")
def test_conflict(self):
"""Test a dataset with conflicting short names."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
dataset["d"] = StructureType("d")
dataset["d"]["c"] = BaseType("c")
projection = [[("c", ())]]
with self.assertRaises(ConstraintExpressionError):
fix_shorthand(projection, dataset)
def nested_object(nested_data):
name = 'nameless'
dataset = DatasetType(name)
seq = dataset['nested'] = SequenceType('nested')
for var in ['a', 'b', 'c']:
seq[var] = BaseType(var)
seq['d'] = SequenceType('d')
for var in ['e', 'f', 'g']:
seq['d'][var] = BaseType(var)
nested = IterData(nested_data, seq)
return nested
def sequence_example():
"""Create a standard sequence from the DAP spec."""
example = SequenceType("example")
example["index"] = BaseType("index")
example["temperature"] = BaseType("temperature")
example["site"] = BaseType("site")
example.data = np.rec.fromrecords([
(10, 15.2, "Diamond_St"),
(11, 13.1, 'Blacktail_Loop'),
(12, 13.3, 'Platinum_St'),
(13, 12.1, 'Kodiak_Trail')], names=list(example.keys()))
return example
def setUp(self):
"""Create a flat IterData."""
template = SequenceType("a")
template["b"] = BaseType("b")
template["c"] = BaseType("c")
template["d"] = BaseType("d")
self.data = IterData([(1, 2, 3), (4, 5, 6)], template)
self.array = np.array(
np.rec.fromrecords([
(1, 2, 3),
(4, 5, 6),
], names=["b", "c", "d"]))
def sequence_type_data():
"""
Simple sequence test data
"""
data = [(10, 15.2, 'Diamond_St'), (11, 13.1, 'Blacktail_Loop'),
(12, 13.3, 'Platinum_St'), (13, 12.1, 'Kodiak_Trail')]
dtype = [('index', '<i4'), ('temperature', '<f8'), ('station', 'S40')]
seq = SequenceType('sequence')
seq['index'] = BaseType('index')
seq['temperature'] = BaseType('temperature')
seq['station'] = BaseType('station')
seq.data = np.array(data, dtype=dtype)
return seq
def demo_dataset():
"""
@Brief Example methods for creating a dataset
http://pydap.org/developer.html#the-dap-data-model
"""
#Create a dataset object
ds = DatasetType(name='Mine')
#Add Some attributes
ds.attributes['history'] = 'David made a dataset'
ds.attributes['conventions'] = 'OOIs special format'
# Create some data and put it in a variable
varname = 'var1'
data = (1, 2, 3, 4, 5, 8)
shape = (8, )
type = Int32 #
dims = ('time', )
attributes = {'long_name': 'long variable name one'}
ds[varname] = BaseType(name=varname,
data=data,
shape=shape,
dimensions=dims,
type=type,
attributes=attributes)
# Now make a grid data object
g = GridType(name='g')
data = numpy.arange(6.)
data.shape = (2, 3)
# The name in the dictionary must match the name in the basetype
g['a'] = BaseType(name='a',
data=data,
shape=data.shape,
type=Float32,
dimensions=('x', 'y'))
g['x'] = BaseType(name='x',
data=numpy.arange(2.),
shape=(2, ),
type=Float64)
g['y'] = BaseType(name='y',
data=numpy.arange(3.),
shape=(3, ),
type=Float64)
ds[g.name] = g
return ds
def test_id(self):
"""Test that the dataset id is not propagated."""
dataset = DatasetType("dataset")
child = BaseType("child")
child.id = "error"
dataset["child"] = child
self.assertEqual(child.id, "child")
def make_grid(self, response, name, data, time_data, attrs, time_attrs,
dims, ttype):
grid = GridType(name=name)
grid[name] = BaseType(name=name,
data=data,
type=ttype,
attributes=attrs,
dimensions=dims,
shape=data.shape)
grid[dims[0]] = BaseType(name=dims[0],
data=time_data,
type=time_data.dtype.char,
attributes=time_attrs,
dimensions=dims,
shape=time_data.shape)
return grid
def setUp(self):
# create dataset
dataset = DatasetType("test")
dataset["a.b"] = BaseType("a.b", np.array(1))
# create WSGI app
self.app = BaseHandler(dataset)
def density(dataset, salinity, temperature, pressure):
"""Calculate in-situ density.
This function calculated in-situ density from absolute salinity and
conservative temperature, using the `gsw.rho` function. Returns a new
sequence with the data.
"""
# find sequence
for sequence in walk(dataset, SequenceType):
break
else:
raise ConstraintExpressionError(
'Function "bounds" should be used on a Sequence.')
selection = sequence[salinity.name, temperature.name, pressure.name]
rows = [tuple(row) for row in selection]
data = np.rec.fromrecords(rows,
names=['salinity', 'temperature', 'pressure'])
rho = gsw.rho(data['salinity'], data['temperature'], data['pressure'])
out = SequenceType("result")
out['rho'] = BaseType("rho", units="kg/m**3")
out.data = np.rec.fromrecords(rho.reshape(-1, 1), names=['rho'])
return out
def test_DatasetType_id():
"""Test that the dataset id is not propagated."""
dataset = DatasetType("dataset")
child = BaseType("child")
child.id = "error"
dataset["child"] = child
assert (child.id == "child")
def make_series(self, response, name, data, attrs, ttype):
base_type = BaseType(name=name,
data=data,
type=ttype,
attributes=attrs)
#grid[dims[0]] = BaseType(name=dims[0], data=time_data, type=time_data.dtype.char, attributes=time_attrs, dimensions=dims, shape=time_data.shape)
return base_type
def test_get_var(self):
"""Test that the id is returned properly."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
self.assertEqual(get_var(dataset, 'b.c'), dataset['b']['c'])
def setUp(self):
# create dataset
self.dataset = DatasetType("test")
self.dataset["foo["] = BaseType("foo[", np.array(1))
# create WSGI app
self.app = BaseHandler(self.dataset)
def handle_dds(self, coverage, dataset, fields):
cov = coverage
seq = SequenceType('data')
for name in fields:
# Strip the data. from the field
if name.startswith('data.'):
name = name[5:]
if re.match(r'.*_[a-z0-9]{32}', name):
continue # Let's not do this
try:
context = coverage.get_parameter_context(name)
attrs = self.get_attrs(cov, name)
#grid[name] = BaseType(name=name, type=self.dap_type(context), attributes=attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,))
seq[name] = BaseType(name=name,
type=self.dap_type(context),
attributes=attrs,
shape=(coverage.num_timesteps(), ))
#grid[cov.temporal_parameter_name] = time_base
except Exception:
log.exception('Problem reading cov %s', str(cov))
continue
dataset['data'] = seq
return dataset
def __init__(self, filepath):
BaseHandler.__init__(self)
self.filepath = filepath
try:
with netcdf_file(self.filepath, 'r') as source:
self.additional_headers.append(('Last-modified', (formatdate(
time.mktime(time.localtime(
os.stat(filepath)[ST_MTIME]))))))
# shortcuts
vars = source.variables
dims = source.dimensions
# build dataset
name = os.path.split(filepath)[1]
self.dataset = DatasetType(
name, attributes=dict(NC_GLOBAL=attrs(source)))
for dim in dims:
if dims[dim] is None:
self.dataset.attributes['DODS_EXTRA'] = {
'Unlimited_Dimension': dim,
}
break
# add grids
grids = [var for var in vars if var not in dims]
for grid in grids:
self.dataset[grid] = GridType(grid, attrs(vars[grid]))
# add array
self.dataset[grid][grid] = BaseType(
grid, LazyVariable(source, grid, grid, self.filepath),
vars[grid].dimensions, attrs(vars[grid]))
# add maps
for dim in vars[grid].dimensions:
self.dataset[grid][dim] = BaseType(
dim, vars[dim][:], None, attrs(vars[dim]))
# add dims
for dim in dims:
self.dataset[dim] = BaseType(dim, vars[dim][:], None,
attrs(vars[dim]))
except Exception as exc:
raise
message = 'Unable to open file %s: %s' % (filepath, exc)
raise OpenFileError(message)
def test_delitem(self):
"""Test item deletion."""
var = StructureType("var")
var["one"] = BaseType("one")
self.assertEqual(var.keys(), ['one'])
del var["one"]
self.assertEqual(var.keys(), [])
def test_regexp(self):
sequence = SequenceType("sequence")
sequence["name"] = BaseType("name")
sequence.data = IterData([
("John", "Paul", "George", "Ringo"),
], sequence)
filtered = sequence[ConstraintExpression('sequence.name=~"J.*"')]
self.assertEqual(list(filtered.iterdata()), [("John", )])
def test_BaseType_comparisons():
"""Test that comparisons are applied to data."""
var = BaseType("var", np.array(1))
assert (var == 1)
assert (var != 2)
assert (var >= 0)
assert (var <= 2)
assert (var > 0)
assert (var < 2)
def test_comparisons(self):
"""Test that comparisons are applied to data."""
var = BaseType("var", np.array(1))
self.assertTrue(var == 1)
self.assertTrue(var != 2)
self.assertTrue(var >= 0)
self.assertTrue(var <= 2)
self.assertTrue(var > 0)
self.assertTrue(var < 2)
