def test_conflict(self):
"""Test a dataset with conflicting short names."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
dataset["d"] = StructureType("d")
dataset["d"]["c"] = BaseType("c")
projection = [[("c", ())]]
with self.assertRaises(ConstraintExpressionError):
fix_shorthand(projection, dataset)
def test_conflict(self):
"""Test a dataset with conflicting short names."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
dataset["d"] = StructureType("d")
dataset["d"]["c"] = BaseType("c")
projection = [[("c", ())]]
with self.assertRaises(ConstraintExpressionError):
fix_shorthand(projection, dataset)
def parse(self, projection, selection, buffer_size=BUFFER_SIZE):
"""
Parse the constraint expression.
"""
if self.dataset is None:
raise NotImplementedError(
"Subclasses must define a dataset attribute pointing to a DatasetType.")
# make a copy of the dataset, so we can filter sequences inplace
dataset = self.dataset.clone()
# apply the selection to the dataset, inplace
apply_selection(selection, dataset)
# wrap data in Arrayterator, to optimize projection/selection
#dataset = wrap_arrayterator(dataset, buffer_size)
# fix projection
if projection:
projection = fix_shorthand(projection, dataset)
else:
projection = [[(key, ())] for key in dataset.keys()]
dataset = apply_projection(projection, dataset)
return dataset
def test_fix_projection(self):
"""Test a dataset that can use the shorthand notation."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
projection = [[("c", ())]]
self.assertEqual(fix_shorthand(projection, dataset), [[('b', ()),
('c', ())]])
def test_fix_projection(self):
"""Test a dataset that can use the shorthand notation."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
projection = [[("c", ())]]
self.assertEqual(
fix_shorthand(projection, dataset),
[[('b', ()), ('c', ())]])
def parse(self, projection, selection):
"""
Parse the constraint expression.
"""
if self.dataset is None:
raise NotImplementedError(
"Subclasses must define a dataset attribute pointing to a DatasetType.")
# make a copy of the dataset, so we can filter sequences inplace
dataset = self.dataset.clone()
# apply the selection to the dataset, inplace
apply_selection(selection, dataset)
# fix projection
if projection:
projection = fix_shorthand(projection, dataset)
else:
projection = [[(key, ())] for key in dataset.keys()]
out = apply_projection(projection, dataset)
return out
def parse(self, projection, selection, buffer_size=BUFFER_SIZE):
"""Parse the constraint expression, returning a new dataset."""
if self.dataset is None:
raise NotImplementedError(
"Subclasses must define a ``dataset`` attribute pointing to a"
"``DatasetType`` object.")
# make a copy of the dataset, so we can filter sequences inplace
dataset = copy.copy(self.dataset)
# apply the selection to the dataset, inplace
apply_selection(selection, dataset)
# wrap data in Arrayterator, to optimize projection/selection
dataset = wrap_arrayterator(dataset, buffer_size)
# fix projection
if projection:
projection = fix_shorthand(projection, dataset)
else:
projection = [[(key, ())] for key in dataset.keys()]
dataset = apply_projection(projection, dataset)
return dataset
def parse(self, projection, selection):
"""
Parse the constraint expression.
"""
if self.dataset is None:
raise NotImplementedError(
"Subclasses must define a dataset attribute pointing to a DatasetType."
)
# make a copy of the dataset, so we can filter sequences inplace
dataset = self.dataset.clone()
# apply the selection to the dataset, inplace
apply_selection(selection, dataset)
# fix projection
if projection:
projection = fix_shorthand(projection, dataset)
else:
projection = [[(key, ())] for key in dataset.keys()]
out = apply_projection(projection, dataset)
return out
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
original_query = req.query_string
projection, selection = parse_ce(req.query_string)
# apply selection without any function calls
req.query_string = '&'.join(expr for expr in selection if not FUNCTION.match(expr))
res = req.get_response(self.app)
# ignore DAS requests
path, response = req.path.rsplit('.', 1)
if response == 'das':
return self.app(environ, start_response)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
environ['QUERY_STRING'] = original_query
return self.app(environ, start_response)
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (expr for expr in selection if FUNCTION.match(expr))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<' : operator.lt,
'>' : operator.gt,
'!=': operator.ne,
'=' : operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
data = np.fromiter(sequence)
print data.shape
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
data = np.asarray(list(sequence), 'O')[valid]
sequence.data = np.asarray(list(sequence), 'O')[valid]
dataset = out
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, basestring)]
func = [p for p in projection if isinstance(p, basestring)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(operator.getitem, [out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
projection, selection = parse_ce(req.query_string)
# check if there are any functions calls in the request
called = (
any(s for s in selection if FUNCTION.match(s)) or
any(p for p in projection if isinstance(p, string_types)))
# ignore DAS requests and requests without functions
path, response = req.path.rsplit('.', 1)
if response == 'das' or not called:
return self.app(environ, start_response)
# apply selection without any function calls
req.query_string = '&'.join(
s for s in selection if not FUNCTION.match(s))
res = req.get_response(self.app)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
raise ServerError("Unable to call server-side function!")
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (s for s in selection if FUNCTION.match(s))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<': operator.lt,
'>': operator.gt,
'!=': operator.ne,
'=': operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
child = list(sequence.children())[0]
data = np.fromiter(child.data, child.dtype)
if data.dtype.char == "S":
valid = np.array(
list(map(lambda v: op(str(v), str(other)), data)),
bool)
else:
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
sequence.data = np.rec.fromrecords(
[tuple(row) for row in sequence],
names=sequence.keys())[valid]
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, string_types)]
func = [p for p in projection if isinstance(p, string_types)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(
operator.getitem, [out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
def __call__(self, environ, start_response):
# specify that we want the parsed dataset
environ['x-wsgiorg.want_parsed_response'] = True
req = Request(environ)
projection, selection = parse_ce(req.query_string)
# check if there are any functions calls in the request
called = (any(s for s in selection if FUNCTION.match(s))
or any(p for p in projection if isinstance(p, string_types)))
# ignore DAS requests and requests without functions
path, response = req.path.rsplit('.', 1)
if response == 'das' or not called:
return self.app(environ, start_response)
# apply selection without any function calls
req.query_string = '&'.join(s for s in selection
if not FUNCTION.match(s))
res = req.get_response(self.app)
# get the dataset
method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False)
if not method:
raise ServerError("Unable to call server-side function!")
dataset = method(DatasetType)
# apply selection containing server-side functions
selection = (s for s in selection if FUNCTION.match(s))
for expr in selection:
if RELOP.search(expr):
call, op, other = RELOP.split(expr)
op = {
'<': operator.lt,
'>': operator.gt,
'!=': operator.ne,
'=': operator.eq,
'>=': operator.ge,
'<=': operator.le,
'=~': lambda a, b: re.match(b, a),
}[op]
other = ast.literal_eval(other)
else:
call, op, other = expr, operator.eq, 1
# evaluate the function call
sequence = eval_function(dataset, call, self.functions)
# is this an inplace call?
for var in walk(dataset, SequenceType):
if sequence is var:
break
else:
# get the data from the resulting variable, and use it to
# constrain the original dataset
child = list(sequence.children())[0]
data = np.fromiter(child.data, child.dtype)
if data.dtype.char == "S":
valid = np.array(
list(map(lambda v: op(str(v), str(other)), data)),
bool)
else:
valid = op(data, other)
for sequence in walk(dataset, SequenceType):
sequence.data = np.rec.fromrecords(
[tuple(row) for row in sequence],
names=sequence.keys())[valid]
# now apply projection
if projection:
projection = fix_shorthand(projection, dataset)
base = [p for p in projection if not isinstance(p, string_types)]
func = [p for p in projection if isinstance(p, string_types)]
# apply non-function projection
out = apply_projection(base, dataset)
# apply function projection
for call in func:
var = eval_function(dataset, call, self.functions)
for child in walk(var):
parent = reduce(operator.getitem,
[out] + child.id.split('.')[:-1])
if child.name not in parent.keys():
parent[child.name] = child
break
dataset = out
# Return the original response (DDS, DAS, etc.)
path, response = req.path.rsplit('.', 1)
res = BaseHandler.responses[response](dataset)
return res(environ, start_response)
