def test_conflict(self): """Test a dataset with conflicting short names.""" dataset = DatasetType("a") dataset["b"] = StructureType("b") dataset["b"]["c"] = BaseType("c") dataset["d"] = StructureType("d") dataset["d"]["c"] = BaseType("c") projection = [[("c", ())]] with self.assertRaises(ConstraintExpressionError): fix_shorthand(projection, dataset)
def parse(self, projection, selection, buffer_size=BUFFER_SIZE): """ Parse the constraint expression. """ if self.dataset is None: raise NotImplementedError( "Subclasses must define a dataset attribute pointing to a DatasetType.") # make a copy of the dataset, so we can filter sequences inplace dataset = self.dataset.clone() # apply the selection to the dataset, inplace apply_selection(selection, dataset) # wrap data in Arrayterator, to optimize projection/selection #dataset = wrap_arrayterator(dataset, buffer_size) # fix projection if projection: projection = fix_shorthand(projection, dataset) else: projection = [[(key, ())] for key in dataset.keys()] dataset = apply_projection(projection, dataset) return dataset
def test_fix_projection(self): """Test a dataset that can use the shorthand notation.""" dataset = DatasetType("a") dataset["b"] = StructureType("b") dataset["b"]["c"] = BaseType("c") projection = [[("c", ())]] self.assertEqual(fix_shorthand(projection, dataset), [[('b', ()), ('c', ())]])
def test_fix_projection(self): """Test a dataset that can use the shorthand notation.""" dataset = DatasetType("a") dataset["b"] = StructureType("b") dataset["b"]["c"] = BaseType("c") projection = [[("c", ())]] self.assertEqual( fix_shorthand(projection, dataset), [[('b', ()), ('c', ())]])
def parse(self, projection, selection): """ Parse the constraint expression. """ if self.dataset is None: raise NotImplementedError( "Subclasses must define a dataset attribute pointing to a DatasetType.") # make a copy of the dataset, so we can filter sequences inplace dataset = self.dataset.clone() # apply the selection to the dataset, inplace apply_selection(selection, dataset) # fix projection if projection: projection = fix_shorthand(projection, dataset) else: projection = [[(key, ())] for key in dataset.keys()] out = apply_projection(projection, dataset) return out
def parse(self, projection, selection, buffer_size=BUFFER_SIZE): """Parse the constraint expression, returning a new dataset.""" if self.dataset is None: raise NotImplementedError( "Subclasses must define a ``dataset`` attribute pointing to a" "``DatasetType`` object.") # make a copy of the dataset, so we can filter sequences inplace dataset = copy.copy(self.dataset) # apply the selection to the dataset, inplace apply_selection(selection, dataset) # wrap data in Arrayterator, to optimize projection/selection dataset = wrap_arrayterator(dataset, buffer_size) # fix projection if projection: projection = fix_shorthand(projection, dataset) else: projection = [[(key, ())] for key in dataset.keys()] dataset = apply_projection(projection, dataset) return dataset
def parse(self, projection, selection): """ Parse the constraint expression. """ if self.dataset is None: raise NotImplementedError( "Subclasses must define a dataset attribute pointing to a DatasetType." ) # make a copy of the dataset, so we can filter sequences inplace dataset = self.dataset.clone() # apply the selection to the dataset, inplace apply_selection(selection, dataset) # fix projection if projection: projection = fix_shorthand(projection, dataset) else: projection = [[(key, ())] for key in dataset.keys()] out = apply_projection(projection, dataset) return out
def __call__(self, environ, start_response): # specify that we want the parsed dataset environ['x-wsgiorg.want_parsed_response'] = True req = Request(environ) original_query = req.query_string projection, selection = parse_ce(req.query_string) # apply selection without any function calls req.query_string = '&'.join(expr for expr in selection if not FUNCTION.match(expr)) res = req.get_response(self.app) # ignore DAS requests path, response = req.path.rsplit('.', 1) if response == 'das': return self.app(environ, start_response) # get the dataset method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False) if not method: environ['QUERY_STRING'] = original_query return self.app(environ, start_response) dataset = method(DatasetType) # apply selection containing server-side functions selection = (expr for expr in selection if FUNCTION.match(expr)) for expr in selection: if RELOP.search(expr): call, op, other = RELOP.split(expr) op = { '<' : operator.lt, '>' : operator.gt, '!=': operator.ne, '=' : operator.eq, '>=': operator.ge, '<=': operator.le, '=~': lambda a, b: re.match(b, a), }[op] other = ast.literal_eval(other) else: call, op, other = expr, operator.eq, 1 # evaluate the function call sequence = eval_function(dataset, call, self.functions) # is this an inplace call? for var in walk(dataset, SequenceType): if sequence is var: break else: # get the data from the resulting variable, and use it to # constrain the original dataset data = np.fromiter(sequence) print data.shape valid = op(data, other) for sequence in walk(dataset, SequenceType): data = np.asarray(list(sequence), 'O')[valid] sequence.data = np.asarray(list(sequence), 'O')[valid] dataset = out # now apply projection if projection: projection = fix_shorthand(projection, dataset) base = [p for p in projection if not isinstance(p, basestring)] func = [p for p in projection if isinstance(p, basestring)] # apply non-function projection out = apply_projection(base, dataset) # apply function projection for call in func: var = eval_function(dataset, call, self.functions) for child in walk(var): parent = reduce(operator.getitem, [out] + child.id.split('.')[:-1]) if child.name not in parent.keys(): parent[child.name] = child break dataset = out # Return the original response (DDS, DAS, etc.) path, response = req.path.rsplit('.', 1) res = BaseHandler.responses[response](dataset) return res(environ, start_response)
def __call__(self, environ, start_response): # specify that we want the parsed dataset environ['x-wsgiorg.want_parsed_response'] = True req = Request(environ) projection, selection = parse_ce(req.query_string) # check if there are any functions calls in the request called = ( any(s for s in selection if FUNCTION.match(s)) or any(p for p in projection if isinstance(p, string_types))) # ignore DAS requests and requests without functions path, response = req.path.rsplit('.', 1) if response == 'das' or not called: return self.app(environ, start_response) # apply selection without any function calls req.query_string = '&'.join( s for s in selection if not FUNCTION.match(s)) res = req.get_response(self.app) # get the dataset method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False) if not method: raise ServerError("Unable to call server-side function!") dataset = method(DatasetType) # apply selection containing server-side functions selection = (s for s in selection if FUNCTION.match(s)) for expr in selection: if RELOP.search(expr): call, op, other = RELOP.split(expr) op = { '<': operator.lt, '>': operator.gt, '!=': operator.ne, '=': operator.eq, '>=': operator.ge, '<=': operator.le, '=~': lambda a, b: re.match(b, a), }[op] other = ast.literal_eval(other) else: call, op, other = expr, operator.eq, 1 # evaluate the function call sequence = eval_function(dataset, call, self.functions) # is this an inplace call? for var in walk(dataset, SequenceType): if sequence is var: break else: # get the data from the resulting variable, and use it to # constrain the original dataset child = list(sequence.children())[0] data = np.fromiter(child.data, child.dtype) if data.dtype.char == "S": valid = np.array( list(map(lambda v: op(str(v), str(other)), data)), bool) else: valid = op(data, other) for sequence in walk(dataset, SequenceType): sequence.data = np.rec.fromrecords( [tuple(row) for row in sequence], names=sequence.keys())[valid] # now apply projection if projection: projection = fix_shorthand(projection, dataset) base = [p for p in projection if not isinstance(p, string_types)] func = [p for p in projection if isinstance(p, string_types)] # apply non-function projection out = apply_projection(base, dataset) # apply function projection for call in func: var = eval_function(dataset, call, self.functions) for child in walk(var): parent = reduce( operator.getitem, [out] + child.id.split('.')[:-1]) if child.name not in parent.keys(): parent[child.name] = child break dataset = out # Return the original response (DDS, DAS, etc.) path, response = req.path.rsplit('.', 1) res = BaseHandler.responses[response](dataset) return res(environ, start_response)
def __call__(self, environ, start_response): # specify that we want the parsed dataset environ['x-wsgiorg.want_parsed_response'] = True req = Request(environ) projection, selection = parse_ce(req.query_string) # check if there are any functions calls in the request called = (any(s for s in selection if FUNCTION.match(s)) or any(p for p in projection if isinstance(p, string_types))) # ignore DAS requests and requests without functions path, response = req.path.rsplit('.', 1) if response == 'das' or not called: return self.app(environ, start_response) # apply selection without any function calls req.query_string = '&'.join(s for s in selection if not FUNCTION.match(s)) res = req.get_response(self.app) # get the dataset method = getattr(res.app_iter, 'x_wsgiorg_parsed_response', False) if not method: raise ServerError("Unable to call server-side function!") dataset = method(DatasetType) # apply selection containing server-side functions selection = (s for s in selection if FUNCTION.match(s)) for expr in selection: if RELOP.search(expr): call, op, other = RELOP.split(expr) op = { '<': operator.lt, '>': operator.gt, '!=': operator.ne, '=': operator.eq, '>=': operator.ge, '<=': operator.le, '=~': lambda a, b: re.match(b, a), }[op] other = ast.literal_eval(other) else: call, op, other = expr, operator.eq, 1 # evaluate the function call sequence = eval_function(dataset, call, self.functions) # is this an inplace call? for var in walk(dataset, SequenceType): if sequence is var: break else: # get the data from the resulting variable, and use it to # constrain the original dataset child = list(sequence.children())[0] data = np.fromiter(child.data, child.dtype) if data.dtype.char == "S": valid = np.array( list(map(lambda v: op(str(v), str(other)), data)), bool) else: valid = op(data, other) for sequence in walk(dataset, SequenceType): sequence.data = np.rec.fromrecords( [tuple(row) for row in sequence], names=sequence.keys())[valid] # now apply projection if projection: projection = fix_shorthand(projection, dataset) base = [p for p in projection if not isinstance(p, string_types)] func = [p for p in projection if isinstance(p, string_types)] # apply non-function projection out = apply_projection(base, dataset) # apply function projection for call in func: var = eval_function(dataset, call, self.functions) for child in walk(var): parent = reduce(operator.getitem, [out] + child.id.split('.')[:-1]) if child.name not in parent.keys(): parent[child.name] = child break dataset = out # Return the original response (DDS, DAS, etc.) path, response = req.path.rsplit('.', 1) res = BaseHandler.responses[response](dataset) return res(environ, start_response)