def test_no_ce(self):
data = np.rec.fromrecords(
D1.Drifters.data.tolist(), names=list(D1.Drifters.keys()))
projection, selection = parse_ce('')
dataset = BaseHandler(D1).parse(projection, selection)
np.testing.assert_array_equal(data, dataset.Drifters.data)
def __init__(self, url):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
dds = requests.get(ddsurl).text.encode('utf-8')
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
das = requests.get(dasurl).text.encode('utf-8')
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.descr)
for var in walk(self.dataset, SequenceType):
var.data = SequenceProxy(url, var.id, var.descr)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if index and isinstance(target.data, BaseProxy):
target.data.slice = fix_slice(index, target.shape)
def test_no_ce(self):
data = np.rec.fromrecords(D1.Drifters.data.tolist(),
names=D1.Drifters.keys())
projection, selection = parse_ce('')
dataset = BaseHandler(D1).parse(projection, selection)
np.testing.assert_array_equal(data, dataset.Drifters.data)
def test_complex(self):
"""Test a more complex constraint expression."""
projection, selection = parse_ce("a,b[0:2:9],c&a>1&b<2")
self.assertEqual(
projection,
[[("a", ())], [("b", (slice(0, 10, 2),))], [("c", ())]])
self.assertEqual(selection, ["a>1", "b<2"])
def __call__(self, environ, start_response):
req = Request(environ)
path, response = req.path.rsplit('.', 1)
if response == 'das':
req.query_string = ''
projection, selection = parse_ce(req.query_string)
buffer_size = environ.get('pydap.buffer_size', BUFFER_SIZE)
try:
# build the dataset and pass it to the proper response, returning a
# WSGI app
dataset = self.parse(projection, selection, buffer_size)
app = self.responses[response](dataset)
app.close = self.close
# now build a Response and set additional headers
res = req.get_response(app)
for key, value in self.additional_headers:
res.headers.add(key, value)
# CORS for Javascript requests
if response in CORS_RESPONSES:
res.headers.add('Access-Control-Allow-Origin', '*')
res.headers.add(
'Access-Control-Allow-Headers',
'Origin, X-Requested-With, Content-Type')
return res(environ, start_response)
except:
# should the exception be catched?
if environ.get('x-wsgiorg.throw_errors'):
raise
else:
res = ErrorResponse(info=sys.exc_info())
return res(environ, start_response)
def __init__(self, url):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
r = requests.get(ddsurl)
r.raise_for_status()
dds = r.text.encode('utf-8')
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
r = requests.get(dasurl)
r.raise_for_status()
das = r.text.encode('utf-8')
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.descr)
for var in walk(self.dataset, SequenceType):
var.data = SequenceProxy(url, var.id, var.descr)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if index and isinstance(target.data, BaseProxy):
target.data.slice = fix_slice(index, target.shape)
def test_complex(self):
"""Test a more complex constraint expression."""
projection, selection = parse_ce("a,b[0:2:9],c&a>1&b<2")
self.assertEqual(
projection,
[[("a", ())], [("b", (slice(0, 10, 2),))], [("c", ())]])
self.assertEqual(selection, ["a>1", "b<2"])
def __call__(self, environ, start_response):
req = Request(environ)
path, response = req.path.rsplit('.', 1)
if response == 'das':
req.query_string = ''
projection, selection = parse_ce(req.query_string)
buffer_size = environ.get('pydap.buffer_size', BUFFER_SIZE)
try:
# build the dataset and pass it to the proper response, returning a
# WSGI app
dataset = self.parse(projection, selection, buffer_size)
app = self.responses[response](dataset)
app.close = self.close
# now build a Response and set additional headers
res = req.get_response(app)
for key, value in self.additional_headers:
res.headers.add(key, value)
# CORS for Javascript requests
if response in CORS_RESPONSES:
res.headers.add('Access-Control-Allow-Origin', '*')
res.headers.add('Access-Control-Allow-Headers',
'Origin, X-Requested-With, Content-Type')
return res(environ, start_response)
except:
# should the exception be catched?
if environ.get('x-wsgiorg.throw_errors'):
raise
else:
res = ErrorResponse(info=sys.exc_info())
return res(environ, start_response)
def test_filtering(self):
data = np.rec.fromrecords(DATA, names=self.dataset.Drifters.keys())
filtered = data[ data['longitude'] < 999 ]
projection, selection = parse_ce('Drifters.longitude<999')
dataset = BaseHandler(self.dataset).parse(projection, selection)
np.testing.assert_array_equal(filtered, dataset.Drifters.data)
def test_filtering(self):
data = np.rec.fromrecords(D1.Drifters.data.tolist(),
names=D1.Drifters.keys())
filtered = data[data['longitude'] < 999]
projection, selection = parse_ce('Drifters.longitude<999')
dataset = BaseHandler(D1).parse(projection, selection)
np.testing.assert_array_equal(filtered, dataset.Drifters.data)
def test_function(self):
"""Test a constraint expression with a function call."""
projection, selection = parse_ce(
"time&bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)")
self.assertEqual(projection, [[("time", ())]])
self.assertEqual(
selection,
["bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)"])
def test_function(self):
"""Test a constraint expression with a function call."""
projection, selection = parse_ce(
"time&bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)")
self.assertEqual(projection, [[("time", ())]])
self.assertEqual(
selection,
["bounds(0,360,-90,90,0,500,00Z01JAN1970,00Z04JAN1970)"])
def __init__(self, url, application=None, session=None, output_grid=True):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + '.dds', query, fragment))
r = GET(ddsurl, application, session)
raise_for_status(r)
dds = r.text
dasurl = urlunsplit((scheme, netloc, path + '.das', query, fragment))
r = GET(dasurl, application, session)
raise_for_status(r)
das = r.text
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, '&'.join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url,
var.id,
var.dtype,
var.shape,
application=application,
session=session)
for var in walk(self.dataset, SequenceType):
template = copy.copy(var)
var.data = SequenceProxy(url,
template,
application=application,
session=session)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if isinstance(target, BaseType):
target.data.slice = fix_slice(index, target.shape)
elif isinstance(target, GridType):
index = fix_slice(index, target.array.shape)
target.array.data.slice = index
for s, child in zip(index, target.maps):
target[child].data.slice = (s, )
elif isinstance(target, SequenceType):
target.data.slice = index
# retrieve only main variable for grid types:
for var in walk(self.dataset, GridType):
var.set_output_grid(output_grid)
def __init__(self, url, application=None, session=None, output_grid=True):
# download DDS/DAS
scheme, netloc, path, query, fragment = urlsplit(url)
ddsurl = urlunsplit((scheme, netloc, path + ".dds", query, fragment))
r = GET(ddsurl, application, session)
raise_for_status(r)
dds = r.text
dasurl = urlunsplit((scheme, netloc, path + ".das", query, fragment))
r = GET(dasurl, application, session)
raise_for_status(r)
das = r.text
# build the dataset from the DDS and add attributes from the DAS
self.dataset = build_dataset(dds)
add_attributes(self.dataset, parse_das(das))
# remove any projection from the url, leaving selections
projection, selection = parse_ce(query)
url = urlunsplit((scheme, netloc, path, "&".join(selection), fragment))
# now add data proxies
for var in walk(self.dataset, BaseType):
var.data = BaseProxy(url, var.id, var.dtype, var.shape, application=application, session=session)
for var in walk(self.dataset, SequenceType):
template = copy.copy(var)
var.data = SequenceProxy(url, template, application=application, session=session)
# apply projections
for var in projection:
target = self.dataset
while var:
token, index = var.pop(0)
target = target[token]
if isinstance(target, BaseType):
target.data.slice = fix_slice(index, target.shape)
elif isinstance(target, GridType):
index = fix_slice(index, target.array.shape)
target.array.data.slice = index
for s, child in zip(index, target.maps):
target[child].data.slice = (s,)
elif isinstance(target, SequenceType):
target.data.slice = index
# retrieve only main variable for grid types:
for var in walk(self.dataset, GridType):
var.set_output_grid(output_grid)
def __call__(self, environ, start_response):
req = Request(environ)
path, response = req.path.rsplit('.', 1)
if response == 'das':
req.query_string = ''
projection, selection = parse_ce(req.query_string)
try:
# build the dataset and pass it to the proper response, returning a
# WSGI app
dataset = self.parse(projection, selection)
app = self.responses[response](dataset)
app.close = self.close
# now build a Response and set additional headers
res = req.get_response(app)
for key, value in self.additional_headers:
res.headers.add(key, value)
return res(environ, start_response)
except HTTPException, exc:
# HTTP exceptions are used to redirect the user
return exc(environ, start_response)
def __call__(self, environ, start_response):
req = Request(environ)
path, response = req.path.rsplit('.', 1)
if response == 'das':
req.query_string = ''
projection, selection = parse_ce(req.query_string)
try:
# build the dataset and pass it to the proper response, returning a
# WSGI app
dataset = self.parse(projection, selection)
app = self.responses[response](dataset)
app.close = self.close
# now build a Response and set additional headers
res = req.get_response(app)
for key, value in self.additional_headers:
res.headers.add(key, value)
return res(environ, start_response)
except HTTPException, exc:
# HTTP exceptions are used to redirect the user
return exc(environ, start_response)
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
projection, selection = parse_ce(req.query_string)
# check if there are any functions calls in the request
called = (any(s for s in selection if FUNCTION.match(s))
or any(p for p in projection if isinstance(p, string_types)))
# ignore DAS requests and requests without functions
path, response = req.path.rsplit('.', 1)
if response == 'das' or not called:
return self.app(environ, start_response)
# apply selection without any function calls
req.query_string = '&'.join(s for s in selection
if not FUNCTION.match(s))
res = req.get_response(self.app)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
raise ServerError("Unable to call server-side function!")
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (s for s in selection if FUNCTION.match(s))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<': operator.lt,
'>': operator.gt,
'!=': operator.ne,
'=': operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
child = list(sequence.children())[0]
data = np.fromiter(child.data, child.dtype)
if data.dtype.char == "S":
valid = np.array(
list(map(lambda v: op(str(v), str(other)), data)),
bool)
else:
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
sequence.data = np.rec.fromrecords(
[tuple(row) for row in sequence],
names=sequence.keys())[valid]
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, string_types)]
func = [p for p in projection if isinstance(p, string_types)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(operator.getitem,
[out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
original_query = req.query_string
projection, selection = parse_ce(req.query_string)
# apply selection without any function calls
req.query_string = '&'.join(expr for expr in selection if not FUNCTION.match(expr))
res = req.get_response(self.app)
# ignore DAS requests
path, response = req.path.rsplit('.', 1)
if response == 'das':
return self.app(environ, start_response)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
environ['QUERY_STRING'] = original_query
return self.app(environ, start_response)
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (expr for expr in selection if FUNCTION.match(expr))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<' : operator.lt,
'>' : operator.gt,
'!=': operator.ne,
'=' : operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
data = np.fromiter(sequence)
print data.shape
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
data = np.asarray(list(sequence), 'O')[valid]
sequence.data = np.asarray(list(sequence), 'O')[valid]
dataset = out
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, basestring)]
func = [p for p in projection if isinstance(p, basestring)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(operator.getitem, [out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def test_only_projection(self):
"""Test a constraint expression with only the projection."""
projection, selection = parse_ce("a,b")
self.assertEqual(projection, [[("a", ())], [("b", ())]])
self.assertEqual(selection, [])
def test_only_selection(self):
"""Test a constraint expression with only the selection."""
projection, selection = parse_ce("c>1")
self.assertEqual(projection, [])
self.assertEqual(selection, ["c>1"])
def test_empty(self):
"""Test no constraint expression."""
projection, selection = parse_ce('')
self.assertEqual(projection, [])
self.assertEqual(selection, [])
def test_basic(self):
"""Test a basic constraint expression."""
projection, selection = parse_ce("a,b&c>1")
self.assertEqual(projection, [[("a", ())], [("b", ())]])
self.assertEqual(selection, ["c>1"])
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
projection, selection = parse_ce(req.query_string)
# check if there are any functions calls in the request
called = (
any(s for s in selection if FUNCTION.match(s)) or
any(p for p in projection if isinstance(p, string_types)))
# ignore DAS requests and requests without functions
path, response = req.path.rsplit('.', 1)
if response == 'das' or not called:
return self.app(environ, start_response)
# apply selection without any function calls
req.query_string = '&'.join(
s for s in selection if not FUNCTION.match(s))
res = req.get_response(self.app)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
raise ServerError("Unable to call server-side function!")
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (s for s in selection if FUNCTION.match(s))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<': operator.lt,
'>': operator.gt,
'!=': operator.ne,
'=': operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
child = list(sequence.children())[0]
data = np.fromiter(child.data, child.dtype)
if data.dtype.char == "S":
valid = np.array(
list(map(lambda v: op(str(v), str(other)), data)),
bool)
else:
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
sequence.data = np.rec.fromrecords(
[tuple(row) for row in sequence],
names=sequence.keys())[valid]
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, string_types)]
func = [p for p in projection if isinstance(p, string_types)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(
operator.getitem, [out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def test_only_selection(self):
"""Test a constraint expression with only the selection."""
projection, selection = parse_ce("c>1")
self.assertEqual(projection, [])
self.assertEqual(selection, ["c>1"])
def test_only_projection(self):
"""Test a constraint expression with only the projection."""
projection, selection = parse_ce("a,b")
self.assertEqual(projection, [[("a", ())], [("b", ())]])
self.assertEqual(selection, [])
def test_basic(self):
"""Test a basic constraint expression."""
projection, selection = parse_ce("a,b&c>1")
self.assertEqual(projection, [[("a", ())], [("b", ())]])
self.assertEqual(selection, ["c>1"])
def test_empty(self):
"""Test no constraint expression."""
projection, selection = parse_ce("")
self.assertEqual(projection, [])
self.assertEqual(selection, [])