def get_var(name, fp, slice_, buf_size=10000):
if name in fp.variables:
var = fp.variables[name]
if hasattr(var, 'set_auto_maskandscale'):
var.set_auto_maskandscale(False)
if var.shape:
data = Arrayterator(var, buf_size)[slice_]
else:
data = numpy.array(get_value(var))
typecode = get_typecode(var)
dims = tuple(quote(dim) for dim in var.dimensions)
attrs = var_attrs(var)
else:
for var in fp.variables:
var = fp.variables[var]
if name in var.dimensions:
size = var.shape[list(var.dimensions).index(name)]
break
data = numpy.arange(size)[slice_]
typecode = data.dtype.char
dims, attrs = (quote(name), ), {}
# handle char vars
if typecode == 'S1':
typecode = 'S'
data = numpy.array([''.join(row) for row in numpy.asarray(data)])
dims = dims[:-1]
return BaseType(name=name,
data=data,
shape=data.shape,
type=typecode,
dimensions=dims,
attributes=attrs)
def get_var(name, fp, slice_, buf_size=10000):
if name in fp.variables:
var = fp.variables[name]
if hasattr(var, 'set_auto_maskandscale'):
var.set_auto_maskandscale(False)
if var.shape:
data = Arrayterator(var, buf_size)[slice_]
else:
data = numpy.array(get_value(var))
typecode = get_typecode(var)
dims = tuple(quote(dim) for dim in var.dimensions)
attrs = var_attrs(var)
else:
for var in fp.variables:
var = fp.variables[var]
if name in var.dimensions:
size = var.shape[
list(var.dimensions).index(name)]
break
data = numpy.arange(size)[slice_]
typecode = data.dtype.char
dims, attrs = (quote(name),), {}
# handle char vars
if typecode == 'S1':
typecode = 'S'
data = numpy.array([''.join(row) for row in numpy.asarray(data)])
dims = dims[:-1]
return BaseType(name=name, data=data, shape=data.shape,
type=typecode, dimensions=dims,
attributes=attrs)
def base(self):
type = self.consume('\w+')
dtype = typemap[type.lower()]
name = quote(self.consume('[^;\[]+'))
shape, dimensions = self.dimensions()
self.consume(';')
var = BaseType(name, dimensions=dimensions)
var.descr = quote(name), dtype, shape
return var
def __init__(self, name, attributes=None, **kwargs):
self.name = quote(name)
self.attributes = attributes or {}
self.attributes.update(kwargs)
# Set the id to the name.
self._id = self.name
def build_attributes(attr, values, level=0):
"""
Recursive function to build the DAS.
"""
# check for metadata
if isinstance(values, dict):
yield '{indent}{attr} {{\n'.format(indent=(level+1)*INDENT, attr=attr)
for k, v in values.items():
for line in build_attributes(k, v, level+1):
yield line
yield '{indent}}}\n'.format(indent=(level+1)*INDENT)
else:
# get type
type = get_type(values)
# encode values
if isinstance(values, basestring) or not isinstance(values, Iterable):
values = [encode(values)]
else:
values = map(encode, values)
yield '{indent}{type} {attr} {values};\n'.format(
indent=(level+1)*INDENT,
type=type,
attr=quote(attr),
values=', '.join(values))
def build_attributes(attr, values, level=0):
"""Recursive function to build the DAS."""
# check for metadata
if isinstance(values, dict):
yield '{indent}{attr} {{\n'.format(indent=(level) * INDENT, attr=attr)
for k, v in values.items():
for line in build_attributes(k, v, level + 1):
yield line
yield '{indent}}}\n'.format(indent=(level) * INDENT)
else:
# get type
type = get_type(values)
# encode values
if (isinstance(values, string_types)
or not isinstance(values, Iterable)
or getattr(values, 'shape', None) == ()):
values = [encode(values)]
else:
values = map(encode, values)
yield '{indent}{type} {attr} {values};\n'.format(
indent=(level) * INDENT,
type=type,
attr=quote(attr),
values=', '.join(values))
def __init__(self, name, attributes=None, **kwargs):
self.name = quote(name)
self.attributes = attributes or {}
self.attributes.update(kwargs)
# Set the id to the name.
self._id = self.name
def get_child(source, target, var, buf_size):
for name, slice_ in var:
if name not in source or name.startswith('_i_'): # hide pytables indexes
break
if isinstance(source[name], (h5py.Dataset, numpy.ndarray)):
dtype = source[name].dtype
if len(dtype):
# array has composite dtype, we need to transform it in a Structure
attrs = dict(getattr(source[name], 'attrs', {}))
target.setdefault(name, StructureType(name=name, attributes=attrs))
target = target[name]
data = source[name]
source = {}
for child in dtype.names:
source[child] = data[child]
else:
# regular array, return data and exit
target[quote(name)] = get_var(name, source, slice_, buf_size)
break
elif name in source and isinstance(source[name], h5py.Group):
# group, return Structure
attrs = dict(source[name].attrs)
target.setdefault(name, StructureType(name=name, attributes=attrs))
target = target[name]
source = source[name]
else:
# when a group is requested by itself, return with all children
for name in source.keys():
get_child(source, target, [(name, ())], buf_size)
def base(self):
"""Parse a base variable, returning a ``BaseType``."""
type = self.consume('\w+')
dtype = typemap[type.lower()]
name = quote(self.consume('[^;\[]+'))
shape, dimensions = self.dimensions()
self.consume(';')
data = DummyData(dtype, shape)
var = BaseType(name, data, dimensions=dimensions)
return var
def base(self):
"""Parse a base variable, returning a ``BaseType``."""
type = self.consume('\w+')
dtype = typemap[type.lower()]
name = quote(self.consume('[^;\[]+'))
shape, dimensions = self.dimensions()
self.consume(';')
data = DummyData(dtype, shape)
var = BaseType(name, data, dimensions=dimensions)
return var
def __setitem__(self, key, item):
key = quote(key)
if key != item.name:
raise KeyError('Key "%s" is different from variable name "%s"!' %
(key, item.name))
if key in self._keys:
self._keys.pop(self._keys.index(key))
self._keys.append(key)
self._dict[key] = item
# Set item id.
item.id = '%s.%s' % (self.id, item.name)
def __setitem__(self, key, item):
key = quote(key)
if key != item.name:
raise KeyError('Key "%s" is different from variable name "%s"!' %
(key, item.name))
if key in self._keys:
self._keys.pop(self._keys.index(key))
self._keys.append(key)
self._dict[key] = item
# Set item id.
item.id = '%s.%s' % (self.id, item.name)
def parse_constraints(self, environ):
buf_size = int(environ.get('pydap.handlers.netcdf.buf_size', 10000))
try:
fp = nc(self.filepath)
except:
message = 'Unable to open file %s.' % self.filepath
raise OpenFileError(message)
last_modified = formatdate(
time.mktime(time.localtime(os.stat(self.filepath)[ST_MTIME])))
environ['pydap.headers'].append(('Last-modified', last_modified))
dataset = DatasetType(name=os.path.split(self.filepath)[1],
attributes={'NC_GLOBAL': var_attrs(fp)})
for dim in fp.dimensions:
if fp.dimensions[dim] is None:
dataset.attributes['DODS_EXTRA'] = {'Unlimited_Dimension': dim}
break
fields, queries = environ['pydap.ce']
fields = fields or [[(quote(name), ())] for name in fp.variables]
for var in fields:
target = dataset
while var:
name, slice_ = var.pop(0)
ncname = urllib.unquote(name)
if (ncname in fp.dimensions
or not fp.variables[ncname].dimensions
or target is not dataset):
target[name] = get_var(ncname, fp, slice_, buf_size)
elif var:
attrs = var_attrs(fp.variables[ncname])
target.setdefault(
name, StructureType(name=name, attributes=attrs))
target = target[name]
else: # return grid
attrs = var_attrs(fp.variables[ncname])
grid = target[name] = GridType(name=name, attributes=attrs)
grid[name] = get_var(ncname, fp, slice_, buf_size)
slice_ = list(slice_) + [slice(None)] * (
len(grid.array.shape) - len(slice_))
for dim, dimslice in zip(fp.variables[ncname].dimensions,
slice_):
axis = get_var(dim, fp, dimslice, buf_size)
grid[axis.name] = axis
dataset._set_id()
dataset.close = fp.close
return dataset
def structure(self):
"""Parse a DAP structure, returning a ``StructureType``."""
structure = StructureType('nameless')
self.consume('structure')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
structure[var.name] = var
self.consume('}')
structure.name = quote(self.consume('[^;]+'))
self.consume(';')
return structure
def structure(self):
"""Parse a DAP structure, returning a ``StructureType``."""
structure = StructureType('nameless')
self.consume('structure')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
structure[var.name] = var
self.consume('}')
structure.name = quote(self.consume('[^;]+'))
self.consume(';')
return structure
def sequence(self):
"""Parse a DAS sequence, returning a ``SequenceType``."""
sequence = SequenceType('nameless')
self.consume('sequence')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
sequence[var.name] = var
self.consume('}')
sequence.name = quote(self.consume('[^;]+'))
self.consume(';')
return sequence
def sequence(self):
"""Parse a DAS sequence, returning a ``SequenceType``."""
sequence = SequenceType('nameless')
self.consume('sequence')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
sequence[var.name] = var
self.consume('}')
sequence.name = quote(self.consume('[^;]+'))
self.consume(';')
return sequence
def sequence(self):
sequence = SequenceType('nameless')
self.consume('sequence')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
sequence[var.name] = var
self.consume('}')
sequence.name = quote(self.consume('[^;]+'))
self.consume(';')
sequence.descr = sequence.name, [c.descr for c in sequence.children()], ()
return sequence
def structure(self):
structure = StructureType('nameless')
self.consume('structure')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
structure[var.name] = var
self.consume('}')
structure.name = quote(self.consume('[^;]+'))
self.consume(';')
structure.descr = structure.name, [c.descr for c in structure.children()], ()
return structure
def parse(self):
"""Parse the DAS, returning a dataset."""
dataset = DatasetType('nameless')
self.consume('dataset')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
dataset[var.name] = var
self.consume('}')
dataset.name = quote(self.consume('[^;]+'))
dataset._set_id(dataset.name)
self.consume(';')
return dataset
def parse(self):
"""Parse the DAS, returning a dataset."""
dataset = DatasetType('nameless')
self.consume('dataset')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
dataset[var.name] = var
self.consume('}')
dataset.name = quote(self.consume('[^;]+'))
dataset._set_id(dataset.name)
self.consume(';')
return dataset
def parse_constraints(self, environ):
buf_size = int(environ.get('pydap.handlers.netcdf.buf_size', 10000))
try:
fp = nc(self.filepath)
except:
message = 'Unable to open file %s.' % self.filepath
raise OpenFileError(message)
last_modified = formatdate( time.mktime( time.localtime( os.stat(self.filepath)[ST_MTIME] ) ) )
environ['pydap.headers'].append( ('Last-modified', last_modified) )
dataset = DatasetType(name=os.path.split(self.filepath)[1],
attributes={'NC_GLOBAL': var_attrs(fp)})
for dim in fp.dimensions:
if fp.dimensions[dim] is None:
dataset.attributes['DODS_EXTRA'] = {'Unlimited_Dimension': dim}
break
fields, queries = environ['pydap.ce']
fields = fields or [[(quote(name), ())] for name in fp.variables]
for var in fields:
target = dataset
while var:
name, slice_ = var.pop(0)
ncname = urllib.unquote(name)
if (ncname in fp.dimensions or
not fp.variables[ncname].dimensions or
target is not dataset):
target[name] = get_var(ncname, fp, slice_, buf_size)
elif var:
attrs = var_attrs(fp.variables[ncname])
target.setdefault(name, StructureType(name=name, attributes=attrs))
target = target[name]
else: # return grid
attrs = var_attrs(fp.variables[ncname])
grid = target[name] = GridType(name=name, attributes=attrs)
grid[name] = get_var(ncname, fp, slice_, buf_size)
slice_ = list(slice_) + [slice(None)] * (len(grid.array.shape) - len(slice_))
for dim, dimslice in zip(fp.variables[ncname].dimensions, slice_):
axis = get_var(dim, fp, dimslice, buf_size)
grid[axis.name] = axis
dataset._set_id()
dataset.close = fp.close
return dataset
def parse(self):
dataset = DatasetType('nameless')
self.consume('dataset')
self.consume('{')
while not self.peek('}'):
var = self.declaration()
dataset[var.name] = var
self.consume('}')
dataset.name = quote(self.consume('[^;]+'))
dataset._set_id(dataset.name)
self.consume(';')
dataset.descr = dataset.name, [c.descr for c in dataset.children()], ()
return dataset
def grid(self):
"""Parse a DAP grid, returning a ``GridType``."""
grid = GridType('nameless')
self.consume('grid')
self.consume('{')
self.consume('array')
self.consume(':')
array = self.base()
grid[array.name] = array
self.consume('maps')
self.consume(':')
while not self.peek('}'):
var = self.base()
grid[var.name] = var
self.consume('}')
grid.name = quote(self.consume('[^;]+'))
self.consume(';')
return grid
def grid(self):
"""Parse a DAP grid, returning a ``GridType``."""
grid = GridType('nameless')
self.consume('grid')
self.consume('{')
self.consume('array')
self.consume(':')
array = self.base()
grid[array.name] = array
self.consume('maps')
self.consume(':')
while not self.peek('}'):
var = self.base()
grid[var.name] = var
self.consume('}')
grid.name = quote(self.consume('[^;]+'))
self.consume(';')
return grid
def grid(self):
grid = GridType('nameless')
self.consume('grid')
self.consume('{')
self.consume('array')
self.consume(':')
array = self.base()
grid[array.name] = array
self.consume('maps')
self.consume(':')
while not self.peek('}'):
var = self.base()
grid[var.name] = var
self.consume('}')
grid.name = quote(self.consume('[^;]+'))
self.consume(';')
grid.descr = grid.name, [c.descr for c in grid.children()], ()
return grid
def __getitem__(self, key):
key = quote(key)
return self._dict[key]
def test_quoting(self):
"""Test a simple quoting."""
self.assertEqual(quote("White space"), "White%20space")
def _set_name(self, name):
self._name = quote(name)
def test_quoting(self):
"""Test a simple quoting."""
self.assertEqual(quote("White space"), "White%20space")
def __getitem__(self, key):
key = quote(key)
return self._dict[key]
def test_quoting_period(self):
"""Test if period is also quoted."""
self.assertEqual(quote("Period."), "Period%2E")
def test_quoting_period(self):
"""Test if period is also quoted."""
self.assertEqual(quote("Period."), "Period%2E")
