def parse_selection(expression, dataset):
"""Parse a selection expression into its elements.
This function will parse a selection expression into three tokens: two
variables or values and a comparison operator. Variables are returned as
Pydap objects from a given dataset, while values are parsed using
``ast.literal_eval``.
"""
id1, op, id2 = re.split('(<=|>=|!=|=~|>|<|=)', expression, 1)
op = {
'<=': operator.le,
'>=': operator.ge,
'!=': operator.ne,
'=': operator.eq,
'>': operator.gt,
'<': operator.lt,
}[op]
try:
id1 = get_var(dataset, id1)
except:
id1 = ast.literal_eval(id1)
try:
id2 = get_var(dataset, id2)
except:
id2 = ast.literal_eval(id2)
return id1, op, id2
def parse_selection(expression, dataset):
"""
Parse a selection expression into its elements.
This function will parse a selection expression into three tokens: two
variables or values and a comparison operator. Variables are returned as
Pydap objects from a given dataset, while values are parsed using
`ast.literal_eval`.
"""
id1, op, id2 = re.split('(<=|>=|!=|=~|>|<|=)', expression, 1)
op = {
'<=': operator.le,
'>=': operator.ge,
'!=': operator.ne,
'=': operator.eq,
'>': operator.gt,
'<': operator.lt,
}[op]
try:
id1 = get_var(dataset, id1)
except:
id1 = ast.literal_eval(id1)
try:
id2 = get_var(dataset, id2)
except:
id2 = ast.literal_eval(id2)
return id1, op, id2
def test_get_var(self):
"""Test that the id is returned properly."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
self.assertEqual(get_var(dataset, 'b.c'), dataset['b']['c'])
def test_get_var(self):
"""Test that the id is returned properly."""
dataset = DatasetType("a")
dataset["b"] = StructureType("b")
dataset["b"]["c"] = BaseType("c")
self.assertEqual(get_var(dataset, 'b.c'), dataset['b']['c'])
def record_generator(nc, dst, table):
debug("record_generator() for dataset %s", dst)
vars = [ iter(get_var(dst, table[varname])) for varname in nc.recvars.keys() ]
while True:
for var in vars:
try:
yield var.next()
except StopIteration:
raise
def apply_projection(projection, dataset):
"""
Apply a given projection to a dataset.
The function returns a new dataset object, after applying the projection to
the original dataset.
"""
out = DatasetType(name=dataset.name, attributes=dataset.attributes)
debug('in apply_projection')
for var in projection:
debug("Looping over %s", var)
target, template = out, dataset
while var:
name, slice_ = var.pop(0)
# FIXME: Need more checks for whether the projection is valid or not
candidate = template[name]
# apply slice
if slice_:
debug("Slicing %s with slice %s", name, slice_)
if isinstance(candidate, BaseType):
candidate.data = candidate[slice_]
elif isinstance(candidate, SequenceType):
candidate = candidate[slice_[0]]
elif isinstance(candidate, GridType):
if len(candidate.maps) != len(slice_):
raise HTTPBadRequest("Attempt to slice grid with %d maps with slice (%s) of length %d" %
(len(candidate.maps), slice_, len(slice_)))
candidate = candidate[slice_]
# handle structures
if isinstance(candidate, StructureType):
debug("instance is a StructureType")
# add variable to target
if name not in target.keys():
if var:
# if there are more children to add we need to clear the
# candidate so it has only explicitly added children;
# also, Grids are degenerated into Structures
if isinstance(candidate, GridType):
candidate = StructureType(candidate.name, candidate.attributes)
candidate._keys = []
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
# fix sequence data, including only variables that are in the sequence
for seq in walk(out, SequenceType):
seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data
debug('out of apply_projection()')
return out
def record_generator(nc, dst, table):
debug("record_generator() for dataset %s", dst)
vars = [
iter(get_var(dst, table[varname]))
for varname in nc.recvars.keys()
]
while True:
for var in vars:
try:
yield var.next()
except StopIteration:
raise
def nonrecord_input():
for varname in nc.non_recvars.keys():
debug("Iterator for %s", varname)
dst_var = get_var(self.dataset, var2id[varname]).data
# skip 0-d variables
if not dst_var.shape:
continue
# Make sure that all elements of the list are iterators
for x in dst_var:
yield x
debug("Done with nonrecord input")
def nonrecord_input():
for varname in nc.non_recvars.keys():
debug("Iterator for %s", varname)
dst_var = get_var(self.dataset, var2id[varname]).data
# skip 0-d variables
if not dst_var.shape:
continue
# Make sure that all elements of the list are iterators
for x in dst_var:
yield x
debug("Done with nonrecord input")
def record_generator(nc, dst, table):
logger.debug("record_generator() for dataset %s", dst)
if not nc.recvars:
logger.debug("file has no record variables")
return
vars = [ iter(get_var(dst, table[varname])) for varname in nc.recvars.keys() ]
while True:
for var in vars:
try:
yield var.next()
except StopIteration:
raise
def apply_projection(projection, dataset):
"""Apply a given projection to a dataset.
This function builds and returns a new dataset by adding those variables
that were requested on the projection.
"""
out = DatasetType(name=dataset.name, attributes=dataset.attributes)
# first collect all the variables
for p in projection:
target, template = out, dataset
for i, (name, slice_) in enumerate(p):
candidate = template[name]
# add variable to target
if isinstance(candidate, StructureType):
if name not in target.keys():
if i < len(p) - 1:
# if there are more children to add we need to clear
# the candidate so it has only explicitly added
# children; also, Grids are degenerated into Structures
if isinstance(candidate, GridType):
candidate = StructureType(candidate.name,
candidate.attributes)
candidate._keys = []
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
# fix sequence data to include only variables that are in the sequence
for seq in walk(out, SequenceType):
seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data
# apply slices
for p in projection:
target = out
for name, slice_ in p:
target, parent = target[name], target
if slice_:
if isinstance(target, BaseType):
target.data = target[slice_]
elif isinstance(target, SequenceType):
parent[name] = target[slice_[0]]
elif isinstance(target, GridType):
parent[name] = target[slice_]
else:
raise ConstraintExpressionError("Invalid projection!")
return out
def apply_projection(projection, dataset):
"""Apply a given projection to a dataset.
This function builds and returns a new dataset by adding those variables
that were requested on the projection.
"""
out = DatasetType(name=dataset.name, attributes=dataset.attributes)
# first collect all the variables
for p in projection:
target, template = out, dataset
for i, (name, slice_) in enumerate(p):
candidate = template[name]
# add variable to target
if isinstance(candidate, StructureType):
if name not in target.keys():
if i < len(p) - 1:
# if there are more children to add we need to clear
# the candidate so it has only explicitly added
# children; also, Grids are degenerated into Structures
if isinstance(candidate, GridType):
candidate = StructureType(
candidate.name, candidate.attributes)
candidate._keys = []
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
# fix sequence data to include only variables that are in the sequence
for seq in walk(out, SequenceType):
seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data
# apply slices
for p in projection:
target = out
for name, slice_ in p:
target, parent = target[name], target
if slice_:
if isinstance(target, BaseType):
target.data = target[slice_]
elif isinstance(target, SequenceType):
parent[name] = target[slice_[0]]
elif isinstance(target, GridType):
parent[name] = target[slice_]
else:
raise ConstraintExpressionError("Invalid projection!")
return out
def apply_projection(projection, dataset):
"""
Apply a given projection to a dataset.
The function returns a new dataset object, after applying the projection to
the original dataset.
"""
out = DatasetType(name=dataset.name, attributes=dataset.attributes)
for var in projection:
target, template = out, dataset
while var:
name, slice_ = var.pop(0)
candidate = template[name]
# apply slice
if slice_:
if isinstance(candidate, BaseType):
candidate.data = candidate[slice_]
elif isinstance(candidate, SequenceType):
candidate = candidate[slice_[0]]
elif isinstance(candidate, GridType):
candidate = candidate[slice_]
# handle structures
if isinstance(candidate, StructureType):
# add variable to target
if name not in target.keys():
if var:
# if there are more children to add we need to clear the
# candidate so it has only explicitly added children;
# also, Grids are degenerated into Structures
if isinstance(candidate, GridType):
candidate = StructureType(candidate.name,
candidate.attributes)
candidate._keys = []
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
# fix sequence data, including only variables that are in the sequence
for seq in walk(out, SequenceType):
seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data
return out
def apply_projection(projection, dataset):
"""
Apply a given projection to a dataset.
The function returns a new dataset object, after applying the projection to
the original dataset.
"""
out = DatasetType(name=dataset.name, attributes=dataset.attributes)
for var in projection:
target, template = out, dataset
while var:
name, slice_ = var.pop(0)
candidate = template[name]
# apply slice
if slice_:
if isinstance(candidate, BaseType):
candidate.data = candidate[slice_]
elif isinstance(candidate, SequenceType):
candidate = candidate[slice_[0]]
elif isinstance(candidate, GridType):
candidate = candidate[slice_]
# handle structures
if isinstance(candidate, StructureType):
# add variable to target
if name not in target.keys():
if var:
# if there are more children to add we need to clear the
# candidate so it has only explicitly added children;
# also, Grids are degenerated into Structures
if isinstance(candidate, GridType):
candidate = StructureType(candidate.name, candidate.attributes)
candidate._keys = []
target[name] = candidate
target, template = target[name], template[name]
else:
target[name] = candidate
# fix sequence data, including only variables that are in the sequence
for seq in walk(out, SequenceType):
seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data
return out
