def __call__(self, environ, start_response):
path_info = environ.get('PATH_INFO') or environ.get('SCRIPT_NAME', '')
path_info = path_info.lstrip('/')
path, type_ = path_info.rsplit('.', 1)
# For DAS, we need to clear the constraint expression to get the
# full dataset.
if type_ == 'das': environ['QUERY_STRING'] = ''
# Add some vars to the environment.
environ['pydap.path'] = path
environ['pydap.response'] = type_
environ['pydap.ce'] = parse_qs(environ.get('QUERY_STRING', ''))
environ['pydap.logger'] = logging.getLogger('pydap')
environ['pydap.headers'] = [] # additional headers
try:
# Build the dataset using the proper subclass method and
# pass it to a response (DAS/DDS/etc.) builder.
dataset = self.parse_constraints(environ)
response = self.response_map[type_]
responder = response(dataset)
return responder(environ, start_response)
except HTTPException, exc:
# Some responses (like HTML) raise an HTTPRedirect when the
# form is posted, so we need this here.
return exc(environ, start_response)
def parse_constraints(self, environ):
dataset = copy.deepcopy(self.dataset)
projection, selection = parse_qs(environ.get('QUERY_STRING', ''))
list_of_var = []
if projection:
for var in projection:
var_name = var[len(var)-1][0]
if var_name in list_of_var:
continue
list_of_var.append(var_name)
if var_name in ['lat', 'lon']:
if var_name == 'lon':
dataset[var_name] = BaseType(name=var_name,
data=self.lons,
shape=self.lons.shape,
dimensions=('lon',),
type=self.lons.dtype.char)
elif var_name == 'lat':
dataset[var_name] = BaseType(name=var_name,
data=self.lats,
shape=self.lats.shape,
dimensions=('lat',),
type=self.lats.dtype.char)
else:
data = self.get_data_for_parameter(var_name, None)
dataset[var_name] = BaseType(name=var_name,
data=data,
shape=data.shape,
dimensions=('lat', 'lon'),
type=data.dtype.char)
return constrain(dataset, environ.get('QUERY_STRING', ''))
def parse_constraints(self, environ):
dataset = copy.deepcopy(self.dataset)
projection, selection = parse_qs(environ.get('QUERY_STRING', ''))
list_of_var = []
if projection:
for var in projection:
var_name = var[len(var) - 1][0]
if var_name in list_of_var:
continue
list_of_var.append(var_name)
if var_name in ['lat', 'lon']:
if var_name == 'lon':
dataset[var_name] = BaseType(name=var_name,
data=self.lons,
shape=self.lons.shape,
dimensions=('lon', ),
type=self.lons.dtype.char)
elif var_name == 'lat':
dataset[var_name] = BaseType(name=var_name,
data=self.lats,
shape=self.lats.shape,
dimensions=('lat', ),
type=self.lats.dtype.char)
else:
data = self.get_data_for_parameter(var_name, None)
dataset[var_name] = BaseType(name=var_name,
data=data,
shape=data.shape,
dimensions=('lat', 'lon'),
type=data.dtype.char)
return constrain(dataset, environ.get('QUERY_STRING', ''))
def __call__(self, environ, start_response):
path_info = environ.get('PATH_INFO') or environ.get('SCRIPT_NAME', '')
path_info = path_info.lstrip('/')
path, type_ = path_info.rsplit('.', 1)
# For DAS, we need to clear the constraint expression to get the
# full dataset.
if type_ == 'das': environ['QUERY_STRING'] = ''
# Add some vars to the environment.
environ['pydap.path'] = path
environ['pydap.response'] = type_
environ['pydap.ce'] = parse_qs(environ.get('QUERY_STRING', ''))
environ['pydap.logger'] = logging.getLogger('pydap')
environ['pydap.headers'] = [] # additional headers
try:
# Build the dataset using the proper subclass method and
# pass it to a response (DAS/DDS/etc.) builder.
dataset = self.parse_constraints(environ)
response = self.response_map[type_]
responder = response(dataset)
return responder(environ, start_response)
except HTTPException, exc:
# Some responses (like HTML) raise an HTTPRedirect when the
# form is posted, so we need this here.
return exc(environ, start_response)
def parse_constraints(self, environ):
projection, selection = parse_qs(environ.get('QUERY_STRING', ''))
if projection:
try:
if self.filepath.endswith('gz'):
file = gzip.open(self.filepath, 'rb')
else:
file = open(self.filepath, "rb")
bytes_read = np.frombuffer(file.read(), np.uint8)
file.close()
except Exception, exc:
message = 'Unable to open file %s: %s' % (self.filepath, exc)
raise OpenFileError(message)
for var in projection:
var_name = var[len(var)-1][0]
slices = var[len(var)-1][1]
if var_name in ['lat', 'lon', 'part_of_day']:
if var_name == 'lon':
if len(slices):
lon_slice = slices[0]
else:
lon_slice = slice(0, 1440, 1)
self.dataset['lon'].data = self.read_variable_lon(lon_slice)
elif var_name == 'lat':
if len(slices):
lat_slice = slices[0]
else:
lat_slice = slice(0, 720, 1)
self.dataset['lat'].data = self.read_variable_lat(lat_slice)
elif var_name == 'part_of_day':
if len(slices):
part_slice = slices[0]
else:
part_slice = slice(0, 2, 1)
self.dataset['part_of_day'].data = self.read_variable_part(part_slice)
else:
for variable in self.variables:
if variable.name == var_name:
slices = var[len(var)-1][1]
if len(slices) != 3:
slices = [slice(0, 1440, 1), slice(0, 720, 1), slice(0, 2, 1)]
# raise ValueError('Cannot obtain slices for %s. '
# 'Should be 3 slices, but %d found' % (var_name, len(slices)))
print 'retrieving %s' % var_name, slices
index = 0
for i in range(len(self.variables)):
if self.variables[i].name == variable.name:
index = i
self.dataset[variable.name]['lon'].data = self.read_variable_lon(slices[0])
self.dataset[variable.name]['lat'].data = self.read_variable_lat(slices[1])
self.dataset[variable.name]['part_of_day'].data = self.read_variable_part(slices[2])
self.dataset[variable.name][variable.name].data = self.read_variable_data(bytes_read, index, slices)
def open_url(url):
"""
Open a given dataset URL, trying different response methods.
The function checks the stub DDX method, and falls back to the
DDS+DAS responses. It can be easily extended for other representations
like JSON.
The URL should point to the dataset, omitting any response extensions
like ``.dds``. Username and password can be passed in the URL like::
http://user:[email protected]:port/path
They will be transmitted as plaintext if the server supports only
Basic authentication, so be careful. For Digest authentication this
is safe.
The URL can point directly to an Opendap dataset, or it can contain
any number of contraint expressions (selection/projections)::
http://example.com/dataset?var1,var2&var3>10
You can also specify a cache directory, a timeout and a proxy using
the global variables from ``pydap.lib``::
>>> import pydap.lib
>>> pydap.lib.TIMEOUT = 60 # seconds
>>> pydap.lib.CACHE = '.cache'
>>> import httplib2
>>> from pydap.util import socks
>>> pydap.lib.PROXY = httplib2.ProxyInfo(socks.PROXY_TYPE_HTTP, 'localhost', 8000)
"""
for response in [_ddx, _ddsdas]:
dataset = response(url)
if dataset: break
else:
raise ClientError("Unable to open dataset.")
# Remove any projections from the url, leaving selections.
scheme, netloc, path, query, fragment = urlsplit(url)
projection, selection = parse_qs(query)
url = urlunsplit(
(scheme, netloc, path, '&'.join(selection), fragment))
# Set data to a Proxy object for BaseType and SequenceType. These
# variables can then be sliced to retrieve the data on-the-fly.
for var in walk(dataset, BaseType):
var.data = ArrayProxy(var.id, url, var.shape)
for var in walk(dataset, SequenceType):
var.data = SequenceProxy(var.id, url)
# Set server-side functions.
dataset.functions = Functions(url)
# Apply the corresponding slices.
projection = fix_shn(projection, dataset)
for var in projection:
target = dataset
while var:
token, slice_ = var.pop(0)
target = target[token]
if slice_ and isinstance(target.data, VariableProxy):
shape = getattr(target, 'shape', (sys.maxint,))
target.data._slice = fix_slice(slice_, shape)
return dataset
def constrain(dataset, ce):
"""
A constraint expression applier.
>>> dataset = DatasetType(name='test')
>>> dataset['seq'] = SequenceType(name='seq')
>>> dataset['seq']['index'] = BaseType(name='index', type=Int32)
>>> dataset['seq']['temperature'] = BaseType(name='temperature', type=Float32)
>>> dataset['seq']['site'] = BaseType(name='site', type=String)
>>> dataset['seq'].data = [
... (10, 17.2, 'Diamont_St'),
... (11, 15.1, 'Blacktail_Loop'),
... (12, 15.3, 'Platinum_St'),
... (13, 15.1, 'Kodiak_Trail')]
>>> for struct_ in dataset.seq:
... print struct_.data
(10, 17.2, 'Diamont_St')
(11, 15.1, 'Blacktail_Loop')
(12, 15.3, 'Platinum_St')
(13, 15.1, 'Kodiak_Trail')
>>> dataset2 = constrain(dataset, 'seq.index>11')
>>> for struct_ in dataset2.seq:
... print struct_.data
(12, 15.3, 'Platinum_St')
(13, 15.1, 'Kodiak_Trail')
>>> dataset2 = constrain(dataset, 'seq.index>11&seq.temperature<15.2')
>>> for struct_ in dataset2.seq:
... print struct_.data
(13, 15.1, 'Kodiak_Trail')
>>> dataset.clear()
>>> dataset['casts'] = SequenceType(name='casts')
>>> dataset['casts']['lat'] = BaseType(name='lat', type=Float32)
>>> dataset['casts']['lon'] = BaseType(name='lon', type=Float32)
>>> dataset['casts']['time'] = BaseType(name='time', type=Float64)
>>> dataset['casts']['profile'] = SequenceType(name='profile')
>>> dataset['casts']['profile']['t'] = BaseType(name='t', type=Float32)
>>> dataset['casts']['profile']['s'] = BaseType(name='s', type=Float32)
>>> dataset['casts']['profile']['p'] = BaseType(name='p', type=Float32)
>>> dataset['casts'].data = [
... (-10.0, 290.0, 1.0, [(21.0, 35.0, 100.0), (20.5, 34.9, 200.0), (19.0, 33.0, 300.0)]),
... (-11.0, 295.0, 2.0, [(22.0, 35.5, 100.0), (21.0, 35.4, 200.0), (20.0, 33.5, 300.0), (19.0, 33.0, 500.0)])]
>>> dataset2 = constrain(dataset, 'casts.lat>-11')
>>> for struct_ in dataset2.casts:
... print struct_.data
(-10.0, 290.0, 1.0, array([[21.0, 35.0, 100.0],
[20.5, 34.9, 200.0],
[19.0, 33.0, 300.0]], dtype=object))
Filtering is guaranteed to work only in outer sequences, but not in inner
sequences like this::
>>> dataset2 = constrain(dataset, 'casts.profile.p>100')
>>> for struct_ in dataset2.casts:
... print struct_.data
(-10.0, 290.0, 1.0, array([[21.0, 35.0, 100.0],
[20.5, 34.9, 200.0],
[19.0, 33.0, 300.0]], dtype=object))
(-11.0, 295.0, 2.0, array([[22.0, 35.5, 100.0],
[21.0, 35.4, 200.0],
[20.0, 33.5, 300.0],
[19.0, 33.0, 500.0]], dtype=object))
Instead, inner sequences have to be filtered inside a loop::
>>> for struct_ in dataset.casts:
... for profile in struct_.profile[ struct_.profile.p > 100 ]:
... print profile.data
(20.5, 34.9, 200.0)
(19.0, 33.0, 300.0)
(21.0, 35.4, 200.0)
(20.0, 33.5, 300.0)
(19.0, 33.0, 500.0)
"""
projection, selection = parse_qs(ce)
projection = projection or [[(key, ())] for key in dataset.keys()]
projection = fix_shn(projection, dataset)
# Make a copy of the dataset.
filtered = copy.deepcopy(dataset)
# Filter sequences.
for seq in walk(filtered, SequenceType):
if seq._nesting_level == 1:
filter_ = []
# Check only selections that apply to the direct children of this sequence
# (ie, skip children from nested sequences).
for cond in [
cond for cond in selection if re.match("%s\.[^\.]+(<=|<|>=|>|=|!=)" % re.escape(seq.id), cond)
]:
id_, op, other = parse_selection(cond, dataset)
filter_.append(op(id_, other))
if filter_:
seq.data = seq[reduce(lambda c1, c2: c1 & c2, filter_)].data
# Create a new empty dataset to build it up.
new_ = DatasetType(name=filtered.name, attributes=filtered.attributes.copy())
for var in projection:
target, template = new_, filtered
while var:
name, slice_ = var.pop(0)
candidate = copy.deepcopy(template[name])
if slice_:
if isinstance(candidate, SequenceType):
candidate = candidate[slice_[0]]
elif isinstance(candidate, BaseType):
candidate.data = candidate[slice_]
candidate.shape = candidate.data.shape
else:
candidate = candidate[slice_]
if isinstance(candidate, StructureType):
if var:
# Convert degenerate grids into structures.
if isinstance(candidate, GridType):
candidate.__class__ = StructureType
candidate.clear()
if name not in target or not var:
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
return new_
def constrain(dataset, ce):
"""
A constraint expression applier.
>>> dataset = DatasetType(name='test')
>>> dataset['seq'] = SequenceType(name='seq')
>>> dataset['seq']['index'] = BaseType(name='index', type=Int32)
>>> dataset['seq']['temperature'] = BaseType(name='temperature', type=Float32)
>>> dataset['seq']['site'] = BaseType(name='site', type=String)
>>> dataset['seq'].data = [
... (10, 17.2, 'Diamont_St'),
... (11, 15.1, 'Blacktail_Loop'),
... (12, 15.3, 'Platinum_St'),
... (13, 15.1, 'Kodiak_Trail')]
>>> for struct_ in dataset.seq:
... print struct_.data
(10, 17.2, 'Diamont_St')
(11, 15.1, 'Blacktail_Loop')
(12, 15.3, 'Platinum_St')
(13, 15.1, 'Kodiak_Trail')
>>> dataset2 = constrain(dataset, 'seq.index>11')
>>> for struct_ in dataset2.seq:
... print struct_.data
(12, 15.3, 'Platinum_St')
(13, 15.1, 'Kodiak_Trail')
>>> dataset2 = constrain(dataset, 'seq.index>11&seq.temperature<15.2')
>>> for struct_ in dataset2.seq:
... print struct_.data
(13, 15.1, 'Kodiak_Trail')
>>> dataset.clear()
>>> dataset['casts'] = SequenceType(name='casts')
>>> dataset['casts']['lat'] = BaseType(name='lat', type=Float32)
>>> dataset['casts']['lon'] = BaseType(name='lon', type=Float32)
>>> dataset['casts']['time'] = BaseType(name='time', type=Float64)
>>> dataset['casts']['profile'] = SequenceType(name='profile')
>>> dataset['casts']['profile']['t'] = BaseType(name='t', type=Float32)
>>> dataset['casts']['profile']['s'] = BaseType(name='s', type=Float32)
>>> dataset['casts']['profile']['p'] = BaseType(name='p', type=Float32)
>>> dataset['casts'].data = [
... (-10.0, 290.0, 1.0, [(21.0, 35.0, 100.0), (20.5, 34.9, 200.0), (19.0, 33.0, 300.0)]),
... (-11.0, 295.0, 2.0, [(22.0, 35.5, 100.0), (21.0, 35.4, 200.0), (20.0, 33.5, 300.0), (19.0, 33.0, 500.0)])]
>>> dataset2 = constrain(dataset, 'casts.lat>-11')
>>> for struct_ in dataset2.casts:
... print struct_.data
(-10.0, 290.0, 1.0, array([[21.0, 35.0, 100.0],
[20.5, 34.9, 200.0],
[19.0, 33.0, 300.0]], dtype=object))
Filtering is guaranteed to work only in outer sequences, but not in inner
sequences like this::
>>> dataset2 = constrain(dataset, 'casts.profile.p>100')
>>> for struct_ in dataset2.casts:
... print struct_.data
(-10.0, 290.0, 1.0, array([[21.0, 35.0, 100.0],
[20.5, 34.9, 200.0],
[19.0, 33.0, 300.0]], dtype=object))
(-11.0, 295.0, 2.0, array([[22.0, 35.5, 100.0],
[21.0, 35.4, 200.0],
[20.0, 33.5, 300.0],
[19.0, 33.0, 500.0]], dtype=object))
Instead, inner sequences have to be filtered inside a loop::
>>> for struct_ in dataset.casts:
... for profile in struct_.profile[ struct_.profile.p > 100 ]:
... print profile.data
(20.5, 34.9, 200.0)
(19.0, 33.0, 300.0)
(21.0, 35.4, 200.0)
(20.0, 33.5, 300.0)
(19.0, 33.0, 500.0)
"""
projection, selection = parse_qs(ce)
projection = projection or [[(key, ())] for key in dataset.keys()]
projection = fix_shn(projection, dataset)
# Make a copy of the dataset.
filtered = copy.deepcopy(dataset)
# Filter sequences.
for seq in walk(filtered, SequenceType):
if seq._nesting_level == 1:
filter_ = []
# Check only selections that apply to the direct children of this sequence
# (ie, skip children from nested sequences).
for cond in [
cond for cond in selection if re.match(
'%s\.[^\.]+(<=|<|>=|>|=|!=)' % re.escape(seq.id), cond)
]:
id_, op, other = parse_selection(cond, dataset)
filter_.append(op(id_, other))
if filter_:
seq.data = seq[reduce(lambda c1, c2: c1 & c2, filter_)].data
# Create a new empty dataset to build it up.
new_ = DatasetType(name=filtered.name,
attributes=filtered.attributes.copy())
for var in projection:
target, template = new_, filtered
while var:
name, slice_ = var.pop(0)
candidate = copy.deepcopy(template[name])
if slice_:
if isinstance(candidate, SequenceType):
candidate = candidate[slice_[0]]
elif isinstance(candidate, BaseType):
candidate.data = candidate[slice_]
candidate.shape = candidate.data.shape
else:
candidate = candidate[slice_]
if isinstance(candidate, StructureType):
if var:
# Convert degenerate grids into structures.
if isinstance(candidate, GridType):
candidate.__class__ = StructureType
candidate.clear()
if name not in target or not var:
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
return new_
