def get_var(name, fp, slice_, buf_size=10000): if name in fp.variables: var = fp.variables[name] if hasattr(var, 'set_auto_maskandscale'): var.set_auto_maskandscale(False) if var.shape: data = Arrayterator(var, buf_size)[slice_] else: data = numpy.array(get_value(var)) typecode = get_typecode(var) dims = tuple(quote(dim) for dim in var.dimensions) attrs = var_attrs(var) else: for var in fp.variables: var = fp.variables[var] if name in var.dimensions: size = var.shape[list(var.dimensions).index(name)] break data = numpy.arange(size)[slice_] typecode = data.dtype.char dims, attrs = (quote(name), ), {} # handle char vars if typecode == 'S1': typecode = 'S' data = numpy.array([''.join(row) for row in numpy.asarray(data)]) dims = dims[:-1] return BaseType(name=name, data=data, shape=data.shape, type=typecode, dimensions=dims, attributes=attrs)
def get_var(name, fp, slice_, buf_size=10000): if name in fp.variables: var = fp.variables[name] if hasattr(var, 'set_auto_maskandscale'): var.set_auto_maskandscale(False) if var.shape: data = Arrayterator(var, buf_size)[slice_] else: data = numpy.array(get_value(var)) typecode = get_typecode(var) dims = tuple(quote(dim) for dim in var.dimensions) attrs = var_attrs(var) else: for var in fp.variables: var = fp.variables[var] if name in var.dimensions: size = var.shape[ list(var.dimensions).index(name)] break data = numpy.arange(size)[slice_] typecode = data.dtype.char dims, attrs = (quote(name),), {} # handle char vars if typecode == 'S1': typecode = 'S' data = numpy.array([''.join(row) for row in numpy.asarray(data)]) dims = dims[:-1] return BaseType(name=name, data=data, shape=data.shape, type=typecode, dimensions=dims, attributes=attrs)
def base(self): type = self.consume('\w+') dtype = typemap[type.lower()] name = quote(self.consume('[^;\[]+')) shape, dimensions = self.dimensions() self.consume(';') var = BaseType(name, dimensions=dimensions) var.descr = quote(name), dtype, shape return var
def __init__(self, name, attributes=None, **kwargs): self.name = quote(name) self.attributes = attributes or {} self.attributes.update(kwargs) # Set the id to the name. self._id = self.name
def build_attributes(attr, values, level=0): """ Recursive function to build the DAS. """ # check for metadata if isinstance(values, dict): yield '{indent}{attr} {{\n'.format(indent=(level+1)*INDENT, attr=attr) for k, v in values.items(): for line in build_attributes(k, v, level+1): yield line yield '{indent}}}\n'.format(indent=(level+1)*INDENT) else: # get type type = get_type(values) # encode values if isinstance(values, basestring) or not isinstance(values, Iterable): values = [encode(values)] else: values = map(encode, values) yield '{indent}{type} {attr} {values};\n'.format( indent=(level+1)*INDENT, type=type, attr=quote(attr), values=', '.join(values))
def build_attributes(attr, values, level=0): """Recursive function to build the DAS.""" # check for metadata if isinstance(values, dict): yield '{indent}{attr} {{\n'.format(indent=(level) * INDENT, attr=attr) for k, v in values.items(): for line in build_attributes(k, v, level + 1): yield line yield '{indent}}}\n'.format(indent=(level) * INDENT) else: # get type type = get_type(values) # encode values if (isinstance(values, string_types) or not isinstance(values, Iterable) or getattr(values, 'shape', None) == ()): values = [encode(values)] else: values = map(encode, values) yield '{indent}{type} {attr} {values};\n'.format( indent=(level) * INDENT, type=type, attr=quote(attr), values=', '.join(values))
def get_child(source, target, var, buf_size): for name, slice_ in var: if name not in source or name.startswith('_i_'): # hide pytables indexes break if isinstance(source[name], (h5py.Dataset, numpy.ndarray)): dtype = source[name].dtype if len(dtype): # array has composite dtype, we need to transform it in a Structure attrs = dict(getattr(source[name], 'attrs', {})) target.setdefault(name, StructureType(name=name, attributes=attrs)) target = target[name] data = source[name] source = {} for child in dtype.names: source[child] = data[child] else: # regular array, return data and exit target[quote(name)] = get_var(name, source, slice_, buf_size) break elif name in source and isinstance(source[name], h5py.Group): # group, return Structure attrs = dict(source[name].attrs) target.setdefault(name, StructureType(name=name, attributes=attrs)) target = target[name] source = source[name] else: # when a group is requested by itself, return with all children for name in source.keys(): get_child(source, target, [(name, ())], buf_size)
def base(self): """Parse a base variable, returning a ``BaseType``.""" type = self.consume('\w+') dtype = typemap[type.lower()] name = quote(self.consume('[^;\[]+')) shape, dimensions = self.dimensions() self.consume(';') data = DummyData(dtype, shape) var = BaseType(name, data, dimensions=dimensions) return var
def __setitem__(self, key, item): key = quote(key) if key != item.name: raise KeyError('Key "%s" is different from variable name "%s"!' % (key, item.name)) if key in self._keys: self._keys.pop(self._keys.index(key)) self._keys.append(key) self._dict[key] = item # Set item id. item.id = '%s.%s' % (self.id, item.name)
def parse_constraints(self, environ): buf_size = int(environ.get('pydap.handlers.netcdf.buf_size', 10000)) try: fp = nc(self.filepath) except: message = 'Unable to open file %s.' % self.filepath raise OpenFileError(message) last_modified = formatdate( time.mktime(time.localtime(os.stat(self.filepath)[ST_MTIME]))) environ['pydap.headers'].append(('Last-modified', last_modified)) dataset = DatasetType(name=os.path.split(self.filepath)[1], attributes={'NC_GLOBAL': var_attrs(fp)}) for dim in fp.dimensions: if fp.dimensions[dim] is None: dataset.attributes['DODS_EXTRA'] = {'Unlimited_Dimension': dim} break fields, queries = environ['pydap.ce'] fields = fields or [[(quote(name), ())] for name in fp.variables] for var in fields: target = dataset while var: name, slice_ = var.pop(0) ncname = urllib.unquote(name) if (ncname in fp.dimensions or not fp.variables[ncname].dimensions or target is not dataset): target[name] = get_var(ncname, fp, slice_, buf_size) elif var: attrs = var_attrs(fp.variables[ncname]) target.setdefault( name, StructureType(name=name, attributes=attrs)) target = target[name] else: # return grid attrs = var_attrs(fp.variables[ncname]) grid = target[name] = GridType(name=name, attributes=attrs) grid[name] = get_var(ncname, fp, slice_, buf_size) slice_ = list(slice_) + [slice(None)] * ( len(grid.array.shape) - len(slice_)) for dim, dimslice in zip(fp.variables[ncname].dimensions, slice_): axis = get_var(dim, fp, dimslice, buf_size) grid[axis.name] = axis dataset._set_id() dataset.close = fp.close return dataset
def structure(self): """Parse a DAP structure, returning a ``StructureType``.""" structure = StructureType('nameless') self.consume('structure') self.consume('{') while not self.peek('}'): var = self.declaration() structure[var.name] = var self.consume('}') structure.name = quote(self.consume('[^;]+')) self.consume(';') return structure
def sequence(self): """Parse a DAS sequence, returning a ``SequenceType``.""" sequence = SequenceType('nameless') self.consume('sequence') self.consume('{') while not self.peek('}'): var = self.declaration() sequence[var.name] = var self.consume('}') sequence.name = quote(self.consume('[^;]+')) self.consume(';') return sequence
def sequence(self): sequence = SequenceType('nameless') self.consume('sequence') self.consume('{') while not self.peek('}'): var = self.declaration() sequence[var.name] = var self.consume('}') sequence.name = quote(self.consume('[^;]+')) self.consume(';') sequence.descr = sequence.name, [c.descr for c in sequence.children()], () return sequence
def structure(self): structure = StructureType('nameless') self.consume('structure') self.consume('{') while not self.peek('}'): var = self.declaration() structure[var.name] = var self.consume('}') structure.name = quote(self.consume('[^;]+')) self.consume(';') structure.descr = structure.name, [c.descr for c in structure.children()], () return structure
def parse(self): """Parse the DAS, returning a dataset.""" dataset = DatasetType('nameless') self.consume('dataset') self.consume('{') while not self.peek('}'): var = self.declaration() dataset[var.name] = var self.consume('}') dataset.name = quote(self.consume('[^;]+')) dataset._set_id(dataset.name) self.consume(';') return dataset
def parse_constraints(self, environ): buf_size = int(environ.get('pydap.handlers.netcdf.buf_size', 10000)) try: fp = nc(self.filepath) except: message = 'Unable to open file %s.' % self.filepath raise OpenFileError(message) last_modified = formatdate( time.mktime( time.localtime( os.stat(self.filepath)[ST_MTIME] ) ) ) environ['pydap.headers'].append( ('Last-modified', last_modified) ) dataset = DatasetType(name=os.path.split(self.filepath)[1], attributes={'NC_GLOBAL': var_attrs(fp)}) for dim in fp.dimensions: if fp.dimensions[dim] is None: dataset.attributes['DODS_EXTRA'] = {'Unlimited_Dimension': dim} break fields, queries = environ['pydap.ce'] fields = fields or [[(quote(name), ())] for name in fp.variables] for var in fields: target = dataset while var: name, slice_ = var.pop(0) ncname = urllib.unquote(name) if (ncname in fp.dimensions or not fp.variables[ncname].dimensions or target is not dataset): target[name] = get_var(ncname, fp, slice_, buf_size) elif var: attrs = var_attrs(fp.variables[ncname]) target.setdefault(name, StructureType(name=name, attributes=attrs)) target = target[name] else: # return grid attrs = var_attrs(fp.variables[ncname]) grid = target[name] = GridType(name=name, attributes=attrs) grid[name] = get_var(ncname, fp, slice_, buf_size) slice_ = list(slice_) + [slice(None)] * (len(grid.array.shape) - len(slice_)) for dim, dimslice in zip(fp.variables[ncname].dimensions, slice_): axis = get_var(dim, fp, dimslice, buf_size) grid[axis.name] = axis dataset._set_id() dataset.close = fp.close return dataset
def parse(self): dataset = DatasetType('nameless') self.consume('dataset') self.consume('{') while not self.peek('}'): var = self.declaration() dataset[var.name] = var self.consume('}') dataset.name = quote(self.consume('[^;]+')) dataset._set_id(dataset.name) self.consume(';') dataset.descr = dataset.name, [c.descr for c in dataset.children()], () return dataset
def grid(self): """Parse a DAP grid, returning a ``GridType``.""" grid = GridType('nameless') self.consume('grid') self.consume('{') self.consume('array') self.consume(':') array = self.base() grid[array.name] = array self.consume('maps') self.consume(':') while not self.peek('}'): var = self.base() grid[var.name] = var self.consume('}') grid.name = quote(self.consume('[^;]+')) self.consume(';') return grid
def grid(self): grid = GridType('nameless') self.consume('grid') self.consume('{') self.consume('array') self.consume(':') array = self.base() grid[array.name] = array self.consume('maps') self.consume(':') while not self.peek('}'): var = self.base() grid[var.name] = var self.consume('}') grid.name = quote(self.consume('[^;]+')) self.consume(';') grid.descr = grid.name, [c.descr for c in grid.children()], () return grid
def __getitem__(self, key): key = quote(key) return self._dict[key]
def test_quoting(self): """Test a simple quoting.""" self.assertEqual(quote("White space"), "White%20space")
def _set_name(self, name): self._name = quote(name)
def test_quoting_period(self): """Test if period is also quoted.""" self.assertEqual(quote("Period."), "Period%2E")