def density(dataset, salinity, temperature, pressure):
"""Calculate in-situ density.
This function calculated in-situ density from absolute salinity and
conservative temperature, using the `gsw.rho` function. Returns a new
sequence with the data.
"""
# find sequence
for sequence in walk(dataset, SequenceType):
break
else:
raise ConstraintExpressionError(
'Function "bounds" should be used on a Sequence.')
selection = sequence[salinity.name, temperature.name, pressure.name]
rows = [tuple(row) for row in selection]
data = np.rec.fromrecords(rows,
names=['salinity', 'temperature', 'pressure'])
rho = gsw.rho(data['salinity'], data['temperature'], data['pressure'])
out = SequenceType("result")
out['rho'] = BaseType("rho", units="kg/m**3")
out.data = np.rec.fromrecords(rho.reshape(-1, 1), names=['rho'])
return out
def metadata_index_responder(sesh, network, climo=False):
'''The function creates a pydap csv response which lists variable metadata out of the database. It returns an generator for the contents of the file
:param sesh: database session
:type sesh: sqlalchemy.orm.session.Session
:param network: Name of the network for which variables should be listed
:type network: str
:rtype: generator
'''
maxlen = 256
if climo:
climo_filt = or_(Variable.cell_method.like('%within%'), Variable.cell_method.like('%over%'))
else:
climo_filt = not_(or_(Variable.cell_method.like('%within%'), Variable.cell_method.like('%over%')))
rv = sesh.query(Variable).join(Network).filter(Network.name == network).filter(climo_filt)
a = np.array([(var.name, var.standard_name, var.cell_method, var.unit) for var in rv],
dtype=np.dtype({'names': ['variable', 'standard_name', 'cell_method', 'unit'],
'formats':[(str, maxlen), (str, maxlen), (str, maxlen), (str, maxlen)]})
)
dst = DatasetType('Variable metadata')
seq = SequenceType('variables')
seq['variable'] = BaseType('variable')
seq['standard_name'] = BaseType('standard_name', reference='http://llnl.gov/')
seq['cell_method'] = BaseType('cell_method', reference='http://llnl.gov/')
seq['unit'] = BaseType('unit')
seq.data = a
dst['variables'] = seq
responder = BaseHandler(dst)
environ = {'PATH_INFO': '/variables.foo.ascii', 'REQUEST_METHOD': 'GET'}
return responder(environ, null_start_response)
def density(dataset, salinity, temperature, pressure):
"""Calculate in-situ density.
This function calculated in-situ density from absolute salinity and
conservative temperature, using the `gsw.rho` function. Returns a new
sequence with the data.
"""
# find sequence
for sequence in walk(dataset, SequenceType):
break
else:
raise ConstraintExpressionError(
'Function "bounds" should be used on a Sequence.')
selection = sequence[salinity.name, temperature.name, pressure.name]
rows = [tuple(row) for row in selection]
data = np.rec.fromrecords(
rows, names=['salinity', 'temperature', 'pressure'])
rho = gsw.rho(data['salinity'], data['temperature'], data['pressure'])
out = SequenceType("result")
out['rho'] = BaseType("rho", units="kg/m**3")
out.data = np.rec.fromrecords(rho.reshape(-1, 1), names=['rho'])
return out
def test_regexp(self):
sequence = SequenceType("sequence")
sequence["name"] = BaseType("name")
sequence.data = IterData([
("John", "Paul", "George", "Ringo"),
], sequence)
filtered = sequence[ConstraintExpression('sequence.name=~"J.*"')]
self.assertEqual(list(filtered.iterdata()), [("John", )])
def test_regexp(self):
sequence = SequenceType("sequence")
sequence["name"] = BaseType("name")
sequence.data = IterData([
("John", "Paul", "George", "Ringo"),
], sequence)
filtered = sequence[ConstraintExpression('sequence.name=~"J.*"')]
self.assertEqual(list(filtered.iterdata()), [("John",)])
def nested_object(nested_data):
name = 'nameless'
dataset = DatasetType(name)
seq = dataset['nested'] = SequenceType('nested')
for var in ['a', 'b', 'c']:
seq[var] = BaseType(var)
seq['d'] = SequenceType('d')
for var in ['e', 'f', 'g']:
seq['d'][var] = BaseType(var)
nested = IterData(nested_data, seq)
return nested
def sequence_example():
"""Create a standard sequence from the DAP spec."""
example = SequenceType("example")
example["index"] = BaseType("index")
example["temperature"] = BaseType("temperature")
example["site"] = BaseType("site")
example.data = np.rec.fromrecords([
(10, 15.2, "Diamond_St"),
(11, 13.1, 'Blacktail_Loop'),
(12, 13.3, 'Platinum_St'),
(13, 12.1, 'Kodiak_Trail')], names=list(example.keys()))
return example
def sequence_example():
"""Create a standard sequence from the DAP spec."""
example = SequenceType("example")
example["index"] = BaseType("index")
example["temperature"] = BaseType("temperature")
example["site"] = BaseType("site")
example.data = np.rec.fromrecords([
(10, 15.2, "Diamond_St"),
(11, 13.1, 'Blacktail_Loop'),
(12, 13.3, 'Platinum_St'),
(13, 12.1, 'Kodiak_Trail')], names=list(example.keys()))
return example
def sequence_type_data():
"""
Simple sequence test data
"""
data = [(10, 15.2, 'Diamond_St'), (11, 13.1, 'Blacktail_Loop'),
(12, 13.3, 'Platinum_St'), (13, 12.1, 'Kodiak_Trail')]
dtype = [('index', '<i4'), ('temperature', '<f8'), ('station', 'S40')]
seq = SequenceType('sequence')
seq['index'] = BaseType('index')
seq['temperature'] = BaseType('temperature')
seq['station'] = BaseType('station')
seq.data = np.array(data, dtype=dtype)
return seq
def sequence(self):
"""Parse a DAS sequence, returning a ``SequenceType``."""
sequence = SequenceType('nameless')
self.consume('sequence')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
sequence[var.name] = var
self.consume('}')
sequence.name = quote(self.consume('[^;]+'))
self.consume(';')
return sequence
def sequence(self):
"""Parse a DAS sequence, returning a ``SequenceType``."""
sequence = SequenceType('nameless')
self.consume('sequence')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
sequence[var.name] = var
self.consume('}')
sequence.name = quote(self.consume('[^;]+'))
self.consume(';')
return sequence
def handle_dds(self, coverage, dataset, fields):
cov = coverage
seq = SequenceType('data')
for name in fields:
# Strip the data. from the field
if name.startswith('data.'):
name = name[5:]
if re.match(r'.*_[a-z0-9]{32}', name):
continue # Let's not do this
try:
context = coverage.get_parameter_context(name)
attrs = self.get_attrs(cov, name)
#grid[name] = BaseType(name=name, type=self.dap_type(context), attributes=attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,))
seq[name] = BaseType(name=name,
type=self.dap_type(context),
attributes=attrs,
shape=(coverage.num_timesteps(), ))
#grid[cov.temporal_parameter_name] = time_base
except Exception:
log.exception('Problem reading cov %s', str(cov))
continue
dataset['data'] = seq
return dataset
def sequence_type_data():
"""
Simple sequence test data
"""
data = [(10, 15.2, 'Diamond_St'),
(11, 13.1, 'Blacktail_Loop'),
(12, 13.3, 'Platinum_St'),
(13, 12.1, 'Kodiak_Trail')]
dtype = [('index', '<i4'),
('temperature', '<f8'),
('station', 'S40')]
seq = SequenceType('sequence')
seq['index'] = BaseType('index')
seq['temperature'] = BaseType('temperature')
seq['station'] = BaseType('station')
seq.data = np.array(data, dtype=dtype)
return seq
def double(dataset, var):
"""A dummy function that doubles a value.
The value must be in a sequence. Return a new sequence with the value
doubled.
"""
# sequence is the first variable
sequence = next(dataset.children())
# get a single variable and double its value
selection = sequence[var.name]
rows = [(value * 2, ) for value in selection]
# create output sequence
out = SequenceType("result")
out["double"] = BaseType("double")
out.data = np.rec.fromrecords(rows, names=["double"])
return out
def simple_object(simple_array):
# add sequence and children for each column
name = 'nameless'
dataset = DatasetType(name)
seq = dataset['sequence'] = SequenceType('sequence')
for var in simple_array.dtype.names:
seq[var] = BaseType(var)
obj = IterData([(0, 1, 10.), (1, 2, 20.), (2, 3, 30.), (3, 4, 40.),
(4, 5, 50.), (5, 6, 60.), (6, 7, 70.), (7, 8, 80.)], seq)
return obj
def double(dataset, var):
"""A dummy function that doubles a value.
The value must be in a sequence. Return a new sequence with the value
doubled.
"""
# sequence is the first variable
sequence = next(dataset.children())
# get a single variable and double its value
selection = sequence[var.name]
rows = [(value*2,) for value in selection]
# create output sequence
out = SequenceType("result")
out["double"] = BaseType("double")
out.data = np.rec.fromrecords(rows, names=["double"])
return out
def setUp(self):
"""Create a flat IterData."""
template = SequenceType("a")
template["b"] = BaseType("b")
template["c"] = BaseType("c")
template["d"] = BaseType("d")
self.data = IterData([(1, 2, 3), (4, 5, 6)], template)
self.array = np.array(
np.rec.fromrecords([
(1, 2, 3),
(4, 5, 6),
], names=["b", "c", "d"]))
def get_dataset(self, cov, fields, slices, selectors, dataset, response):
seq = SequenceType('data')
bitmask = self.get_bitmask(cov, fields, slices, selectors)
if self.is_too_large(bitmask, len(fields)):
log.error('Client request too large. \nFields: %s\nSelectors: %s',
fields, selectors)
return
for name in fields:
# Strip the data. from the field
if name.startswith('data.'):
name = name[5:]
pc = cov.get_parameter_context(name)
if re.match(r'.*_[a-z0-9]{32}', name):
continue # Let's not do this
try:
data = self.get_data(cov, name, bitmask)
attrs = self.get_attrs(cov, name)
if isinstance(pc.param_type, QuantityType):
data, dtype = self.filter_data(data)
seq[name] = self.make_series(response, name, data, attrs,
dtype)
elif isinstance(pc.param_type, ConstantType):
data, dtype = self.filter_data(data)
seq[name] = self.make_series(response, name, data, attrs,
dtype)
elif isinstance(pc.param_type, ConstantRangeType):
#start = time.time()
#convert to string
try:
#scalar case
if data.shape == (2, ):
data = np.atleast_1d('_'.join(
[str(data[0]), str(data[1])]))
else:
for i, d in enumerate(data):
f = [str(d[0]), str(d[1])]
data[i] = '_'.join(f)
except Exception, e:
data = np.asanyarray(['None' for d in data])
seq[name] = self.make_series(response, name, data, attrs,
'S')
elif isinstance(pc.param_type, BooleanType):
data, dtype = self.filter_data(data)
seq[name] = self.make_series(response, name, data, attrs,
dtype)
elif isinstance(pc.param_type, CategoryType):
data, dtype = self.filter_data(data)
#start = time.time()
seq[name] = self.make_series(response, name, data, attrs,
dtype)
def _make_model(with_attributes=False, with_data=False):
def maybe(attributes):
return attributes if with_attributes else None
model = DatasetType(
name='dataset_name',
attributes=maybe(attributes('ds', 1))
)
model['sequence_name'] = SequenceType(
name='sequence_name',
attributes=maybe(attributes('seq', 2))
)
for name in ['a', 'b', 'c']:
model['sequence_name'][name] = BaseType(
name=name,
attributes=maybe(attributes(name, 2))
)
return model
def simple_sequence_dataset(metadata, data):
"""
@brief Create a simple dap dataset object from dictionary content
See test_daptools to see the input structure
"""
# Convert metadata and data to a dap dataset
ds = DatasetType(name=metadata['DataSet Name'])
sequence = SequenceType(name='sequence')
for varname, atts in metadata['variables'].items():
var = BaseType(name=varname, \
data=data[varname], \
shape=(len(data[varname]),), \
#dimensions=(varname,), \
type=Int32, \
attributes=atts)
sequence[varname] = var
ds[sequence.name] = sequence
return ds
from ordereddict import OrderedDict
else:
from collections import OrderedDict
import numpy as np
from pydap.model import (DatasetType, BaseType, SequenceType, GridType,
StructureType)
from pydap.handlers.lib import IterData
from pydap.client import open_file
# A very simple sequence: flat and with no strings. This sequence can be mapped
# directly to a Numpy structured array, and can be easily encoded and decoded
# in the DAP spec.
VerySimpleSequence = DatasetType("VerySimpleSequence")
VerySimpleSequence["sequence"] = SequenceType("sequence")
VerySimpleSequence["sequence"]["byte"] = BaseType("byte")
VerySimpleSequence["sequence"]["int"] = BaseType("int")
VerySimpleSequence["sequence"]["float"] = BaseType("float")
VerySimpleSequence["sequence"].data = np.array([
(0, 1, 10.),
(1, 2, 20.),
(2, 3, 30.),
(3, 4, 40.),
(4, 5, 50.),
(5, 6, 60.),
(6, 7, 70.),
(7, 8, 80.),
],
dtype=[('byte', 'b'),
('int', 'i4'),
import numpy as np
from pydap.model import (DatasetType, BaseType,
SequenceType, GridType, StructureType)
from pydap.handlers.lib import IterData
from pydap.client import open_file
from collections import OrderedDict
# A very simple sequence: flat and with no strings. This sequence can be mapped
# directly to a Numpy structured array, and can be easily encoded and decoded
# in the DAP spec.
VerySimpleSequence = DatasetType("VerySimpleSequence")
VerySimpleSequence["sequence"] = SequenceType("sequence")
VerySimpleSequence["sequence"]["byte"] = BaseType("byte")
VerySimpleSequence["sequence"]["int"] = BaseType("int")
VerySimpleSequence["sequence"]["float"] = BaseType("float")
VerySimpleSequence["sequence"].data = np.array([
(0, 1, 10.),
(1, 2, 20.),
(2, 3, 30.),
(3, 4, 40.),
(4, 5, 50.),
(5, 6, 60.),
(6, 7, 70.),
(7, 8, 80.),
], dtype=[('byte', 'b'), ('int', 'i4'), ('float', 'f4')])
