def density(dataset, salinity, temperature, pressure): """Calculate in-situ density. This function calculated in-situ density from absolute salinity and conservative temperature, using the `gsw.rho` function. Returns a new sequence with the data. """ # find sequence for sequence in walk(dataset, SequenceType): break else: raise ConstraintExpressionError( 'Function "bounds" should be used on a Sequence.') selection = sequence[salinity.name, temperature.name, pressure.name] rows = [tuple(row) for row in selection] data = np.rec.fromrecords(rows, names=['salinity', 'temperature', 'pressure']) rho = gsw.rho(data['salinity'], data['temperature'], data['pressure']) out = SequenceType("result") out['rho'] = BaseType("rho", units="kg/m**3") out.data = np.rec.fromrecords(rho.reshape(-1, 1), names=['rho']) return out
def metadata_index_responder(sesh, network, climo=False): '''The function creates a pydap csv response which lists variable metadata out of the database. It returns an generator for the contents of the file :param sesh: database session :type sesh: sqlalchemy.orm.session.Session :param network: Name of the network for which variables should be listed :type network: str :rtype: generator ''' maxlen = 256 if climo: climo_filt = or_(Variable.cell_method.like('%within%'), Variable.cell_method.like('%over%')) else: climo_filt = not_(or_(Variable.cell_method.like('%within%'), Variable.cell_method.like('%over%'))) rv = sesh.query(Variable).join(Network).filter(Network.name == network).filter(climo_filt) a = np.array([(var.name, var.standard_name, var.cell_method, var.unit) for var in rv], dtype=np.dtype({'names': ['variable', 'standard_name', 'cell_method', 'unit'], 'formats':[(str, maxlen), (str, maxlen), (str, maxlen), (str, maxlen)]}) ) dst = DatasetType('Variable metadata') seq = SequenceType('variables') seq['variable'] = BaseType('variable') seq['standard_name'] = BaseType('standard_name', reference='http://llnl.gov/') seq['cell_method'] = BaseType('cell_method', reference='http://llnl.gov/') seq['unit'] = BaseType('unit') seq.data = a dst['variables'] = seq responder = BaseHandler(dst) environ = {'PATH_INFO': '/variables.foo.ascii', 'REQUEST_METHOD': 'GET'} return responder(environ, null_start_response)
def density(dataset, salinity, temperature, pressure): """Calculate in-situ density. This function calculated in-situ density from absolute salinity and conservative temperature, using the `gsw.rho` function. Returns a new sequence with the data. """ # find sequence for sequence in walk(dataset, SequenceType): break else: raise ConstraintExpressionError( 'Function "bounds" should be used on a Sequence.') selection = sequence[salinity.name, temperature.name, pressure.name] rows = [tuple(row) for row in selection] data = np.rec.fromrecords( rows, names=['salinity', 'temperature', 'pressure']) rho = gsw.rho(data['salinity'], data['temperature'], data['pressure']) out = SequenceType("result") out['rho'] = BaseType("rho", units="kg/m**3") out.data = np.rec.fromrecords(rho.reshape(-1, 1), names=['rho']) return out
def test_regexp(self): sequence = SequenceType("sequence") sequence["name"] = BaseType("name") sequence.data = IterData([ ("John", "Paul", "George", "Ringo"), ], sequence) filtered = sequence[ConstraintExpression('sequence.name=~"J.*"')] self.assertEqual(list(filtered.iterdata()), [("John", )])
def test_regexp(self): sequence = SequenceType("sequence") sequence["name"] = BaseType("name") sequence.data = IterData([ ("John", "Paul", "George", "Ringo"), ], sequence) filtered = sequence[ConstraintExpression('sequence.name=~"J.*"')] self.assertEqual(list(filtered.iterdata()), [("John",)])
def nested_object(nested_data): name = 'nameless' dataset = DatasetType(name) seq = dataset['nested'] = SequenceType('nested') for var in ['a', 'b', 'c']: seq[var] = BaseType(var) seq['d'] = SequenceType('d') for var in ['e', 'f', 'g']: seq['d'][var] = BaseType(var) nested = IterData(nested_data, seq) return nested
def sequence_example(): """Create a standard sequence from the DAP spec.""" example = SequenceType("example") example["index"] = BaseType("index") example["temperature"] = BaseType("temperature") example["site"] = BaseType("site") example.data = np.rec.fromrecords([ (10, 15.2, "Diamond_St"), (11, 13.1, 'Blacktail_Loop'), (12, 13.3, 'Platinum_St'), (13, 12.1, 'Kodiak_Trail')], names=list(example.keys())) return example
def sequence_type_data(): """ Simple sequence test data """ data = [(10, 15.2, 'Diamond_St'), (11, 13.1, 'Blacktail_Loop'), (12, 13.3, 'Platinum_St'), (13, 12.1, 'Kodiak_Trail')] dtype = [('index', '<i4'), ('temperature', '<f8'), ('station', 'S40')] seq = SequenceType('sequence') seq['index'] = BaseType('index') seq['temperature'] = BaseType('temperature') seq['station'] = BaseType('station') seq.data = np.array(data, dtype=dtype) return seq
def sequence(self): """Parse a DAS sequence, returning a ``SequenceType``.""" sequence = SequenceType('nameless') self.consume('sequence') self.consume('{') while not self.peek('}'): var = self.declaration() sequence[var.name] = var self.consume('}') sequence.name = quote(self.consume('[^;]+')) self.consume(';') return sequence
def handle_dds(self, coverage, dataset, fields): cov = coverage seq = SequenceType('data') for name in fields: # Strip the data. from the field if name.startswith('data.'): name = name[5:] if re.match(r'.*_[a-z0-9]{32}', name): continue # Let's not do this try: context = coverage.get_parameter_context(name) attrs = self.get_attrs(cov, name) #grid[name] = BaseType(name=name, type=self.dap_type(context), attributes=attrs, dimensions=(time_name,), shape=(coverage.num_timesteps,)) seq[name] = BaseType(name=name, type=self.dap_type(context), attributes=attrs, shape=(coverage.num_timesteps(), )) #grid[cov.temporal_parameter_name] = time_base except Exception: log.exception('Problem reading cov %s', str(cov)) continue dataset['data'] = seq return dataset
def double(dataset, var): """A dummy function that doubles a value. The value must be in a sequence. Return a new sequence with the value doubled. """ # sequence is the first variable sequence = next(dataset.children()) # get a single variable and double its value selection = sequence[var.name] rows = [(value * 2, ) for value in selection] # create output sequence out = SequenceType("result") out["double"] = BaseType("double") out.data = np.rec.fromrecords(rows, names=["double"]) return out
def simple_object(simple_array): # add sequence and children for each column name = 'nameless' dataset = DatasetType(name) seq = dataset['sequence'] = SequenceType('sequence') for var in simple_array.dtype.names: seq[var] = BaseType(var) obj = IterData([(0, 1, 10.), (1, 2, 20.), (2, 3, 30.), (3, 4, 40.), (4, 5, 50.), (5, 6, 60.), (6, 7, 70.), (7, 8, 80.)], seq) return obj
def double(dataset, var): """A dummy function that doubles a value. The value must be in a sequence. Return a new sequence with the value doubled. """ # sequence is the first variable sequence = next(dataset.children()) # get a single variable and double its value selection = sequence[var.name] rows = [(value*2,) for value in selection] # create output sequence out = SequenceType("result") out["double"] = BaseType("double") out.data = np.rec.fromrecords(rows, names=["double"]) return out
def setUp(self): """Create a flat IterData.""" template = SequenceType("a") template["b"] = BaseType("b") template["c"] = BaseType("c") template["d"] = BaseType("d") self.data = IterData([(1, 2, 3), (4, 5, 6)], template) self.array = np.array( np.rec.fromrecords([ (1, 2, 3), (4, 5, 6), ], names=["b", "c", "d"]))
def get_dataset(self, cov, fields, slices, selectors, dataset, response): seq = SequenceType('data') bitmask = self.get_bitmask(cov, fields, slices, selectors) if self.is_too_large(bitmask, len(fields)): log.error('Client request too large. \nFields: %s\nSelectors: %s', fields, selectors) return for name in fields: # Strip the data. from the field if name.startswith('data.'): name = name[5:] pc = cov.get_parameter_context(name) if re.match(r'.*_[a-z0-9]{32}', name): continue # Let's not do this try: data = self.get_data(cov, name, bitmask) attrs = self.get_attrs(cov, name) if isinstance(pc.param_type, QuantityType): data, dtype = self.filter_data(data) seq[name] = self.make_series(response, name, data, attrs, dtype) elif isinstance(pc.param_type, ConstantType): data, dtype = self.filter_data(data) seq[name] = self.make_series(response, name, data, attrs, dtype) elif isinstance(pc.param_type, ConstantRangeType): #start = time.time() #convert to string try: #scalar case if data.shape == (2, ): data = np.atleast_1d('_'.join( [str(data[0]), str(data[1])])) else: for i, d in enumerate(data): f = [str(d[0]), str(d[1])] data[i] = '_'.join(f) except Exception, e: data = np.asanyarray(['None' for d in data]) seq[name] = self.make_series(response, name, data, attrs, 'S') elif isinstance(pc.param_type, BooleanType): data, dtype = self.filter_data(data) seq[name] = self.make_series(response, name, data, attrs, dtype) elif isinstance(pc.param_type, CategoryType): data, dtype = self.filter_data(data) #start = time.time() seq[name] = self.make_series(response, name, data, attrs, dtype)
def _make_model(with_attributes=False, with_data=False): def maybe(attributes): return attributes if with_attributes else None model = DatasetType( name='dataset_name', attributes=maybe(attributes('ds', 1)) ) model['sequence_name'] = SequenceType( name='sequence_name', attributes=maybe(attributes('seq', 2)) ) for name in ['a', 'b', 'c']: model['sequence_name'][name] = BaseType( name=name, attributes=maybe(attributes(name, 2)) ) return model
def simple_sequence_dataset(metadata, data): """ @brief Create a simple dap dataset object from dictionary content See test_daptools to see the input structure """ # Convert metadata and data to a dap dataset ds = DatasetType(name=metadata['DataSet Name']) sequence = SequenceType(name='sequence') for varname, atts in metadata['variables'].items(): var = BaseType(name=varname, \ data=data[varname], \ shape=(len(data[varname]),), \ #dimensions=(varname,), \ type=Int32, \ attributes=atts) sequence[varname] = var ds[sequence.name] = sequence return ds
from ordereddict import OrderedDict else: from collections import OrderedDict import numpy as np from pydap.model import (DatasetType, BaseType, SequenceType, GridType, StructureType) from pydap.handlers.lib import IterData from pydap.client import open_file # A very simple sequence: flat and with no strings. This sequence can be mapped # directly to a Numpy structured array, and can be easily encoded and decoded # in the DAP spec. VerySimpleSequence = DatasetType("VerySimpleSequence") VerySimpleSequence["sequence"] = SequenceType("sequence") VerySimpleSequence["sequence"]["byte"] = BaseType("byte") VerySimpleSequence["sequence"]["int"] = BaseType("int") VerySimpleSequence["sequence"]["float"] = BaseType("float") VerySimpleSequence["sequence"].data = np.array([ (0, 1, 10.), (1, 2, 20.), (2, 3, 30.), (3, 4, 40.), (4, 5, 50.), (5, 6, 60.), (6, 7, 70.), (7, 8, 80.), ], dtype=[('byte', 'b'), ('int', 'i4'),
import numpy as np from pydap.model import (DatasetType, BaseType, SequenceType, GridType, StructureType) from pydap.handlers.lib import IterData from pydap.client import open_file from collections import OrderedDict # A very simple sequence: flat and with no strings. This sequence can be mapped # directly to a Numpy structured array, and can be easily encoded and decoded # in the DAP spec. VerySimpleSequence = DatasetType("VerySimpleSequence") VerySimpleSequence["sequence"] = SequenceType("sequence") VerySimpleSequence["sequence"]["byte"] = BaseType("byte") VerySimpleSequence["sequence"]["int"] = BaseType("int") VerySimpleSequence["sequence"]["float"] = BaseType("float") VerySimpleSequence["sequence"].data = np.array([ (0, 1, 10.), (1, 2, 20.), (2, 3, 30.), (3, 4, 40.), (4, 5, 50.), (5, 6, 60.), (6, 7, 70.), (7, 8, 80.), ], dtype=[('byte', 'b'), ('int', 'i4'), ('float', 'f4')])