def parse_selection(expression, dataset): """Parse a selection expression into its elements. This function will parse a selection expression into three tokens: two variables or values and a comparison operator. Variables are returned as Pydap objects from a given dataset, while values are parsed using ``ast.literal_eval``. """ id1, op, id2 = re.split('(<=|>=|!=|=~|>|<|=)', expression, 1) op = { '<=': operator.le, '>=': operator.ge, '!=': operator.ne, '=': operator.eq, '>': operator.gt, '<': operator.lt, }[op] try: id1 = get_var(dataset, id1) except: id1 = ast.literal_eval(id1) try: id2 = get_var(dataset, id2) except: id2 = ast.literal_eval(id2) return id1, op, id2
def parse_selection(expression, dataset): """ Parse a selection expression into its elements. This function will parse a selection expression into three tokens: two variables or values and a comparison operator. Variables are returned as Pydap objects from a given dataset, while values are parsed using `ast.literal_eval`. """ id1, op, id2 = re.split('(<=|>=|!=|=~|>|<|=)', expression, 1) op = { '<=': operator.le, '>=': operator.ge, '!=': operator.ne, '=': operator.eq, '>': operator.gt, '<': operator.lt, }[op] try: id1 = get_var(dataset, id1) except: id1 = ast.literal_eval(id1) try: id2 = get_var(dataset, id2) except: id2 = ast.literal_eval(id2) return id1, op, id2
def test_get_var(self): """Test that the id is returned properly.""" dataset = DatasetType("a") dataset["b"] = StructureType("b") dataset["b"]["c"] = BaseType("c") self.assertEqual(get_var(dataset, 'b.c'), dataset['b']['c'])
def record_generator(nc, dst, table): debug("record_generator() for dataset %s", dst) vars = [ iter(get_var(dst, table[varname])) for varname in nc.recvars.keys() ] while True: for var in vars: try: yield var.next() except StopIteration: raise
def apply_projection(projection, dataset): """ Apply a given projection to a dataset. The function returns a new dataset object, after applying the projection to the original dataset. """ out = DatasetType(name=dataset.name, attributes=dataset.attributes) debug('in apply_projection') for var in projection: debug("Looping over %s", var) target, template = out, dataset while var: name, slice_ = var.pop(0) # FIXME: Need more checks for whether the projection is valid or not candidate = template[name] # apply slice if slice_: debug("Slicing %s with slice %s", name, slice_) if isinstance(candidate, BaseType): candidate.data = candidate[slice_] elif isinstance(candidate, SequenceType): candidate = candidate[slice_[0]] elif isinstance(candidate, GridType): if len(candidate.maps) != len(slice_): raise HTTPBadRequest("Attempt to slice grid with %d maps with slice (%s) of length %d" % (len(candidate.maps), slice_, len(slice_))) candidate = candidate[slice_] # handle structures if isinstance(candidate, StructureType): debug("instance is a StructureType") # add variable to target if name not in target.keys(): if var: # if there are more children to add we need to clear the # candidate so it has only explicitly added children; # also, Grids are degenerated into Structures if isinstance(candidate, GridType): candidate = StructureType(candidate.name, candidate.attributes) candidate._keys = [] target[name] = candidate target, template = target[name], template[name] else: target[name] = candidate # fix sequence data, including only variables that are in the sequence for seq in walk(out, SequenceType): seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data debug('out of apply_projection()') return out
def nonrecord_input(): for varname in nc.non_recvars.keys(): debug("Iterator for %s", varname) dst_var = get_var(self.dataset, var2id[varname]).data # skip 0-d variables if not dst_var.shape: continue # Make sure that all elements of the list are iterators for x in dst_var: yield x debug("Done with nonrecord input")
def record_generator(nc, dst, table): logger.debug("record_generator() for dataset %s", dst) if not nc.recvars: logger.debug("file has no record variables") return vars = [ iter(get_var(dst, table[varname])) for varname in nc.recvars.keys() ] while True: for var in vars: try: yield var.next() except StopIteration: raise
def apply_projection(projection, dataset): """Apply a given projection to a dataset. This function builds and returns a new dataset by adding those variables that were requested on the projection. """ out = DatasetType(name=dataset.name, attributes=dataset.attributes) # first collect all the variables for p in projection: target, template = out, dataset for i, (name, slice_) in enumerate(p): candidate = template[name] # add variable to target if isinstance(candidate, StructureType): if name not in target.keys(): if i < len(p) - 1: # if there are more children to add we need to clear # the candidate so it has only explicitly added # children; also, Grids are degenerated into Structures if isinstance(candidate, GridType): candidate = StructureType(candidate.name, candidate.attributes) candidate._keys = [] target[name] = candidate target, template = target[name], template[name] else: target[name] = candidate # fix sequence data to include only variables that are in the sequence for seq in walk(out, SequenceType): seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data # apply slices for p in projection: target = out for name, slice_ in p: target, parent = target[name], target if slice_: if isinstance(target, BaseType): target.data = target[slice_] elif isinstance(target, SequenceType): parent[name] = target[slice_[0]] elif isinstance(target, GridType): parent[name] = target[slice_] else: raise ConstraintExpressionError("Invalid projection!") return out
def apply_projection(projection, dataset): """Apply a given projection to a dataset. This function builds and returns a new dataset by adding those variables that were requested on the projection. """ out = DatasetType(name=dataset.name, attributes=dataset.attributes) # first collect all the variables for p in projection: target, template = out, dataset for i, (name, slice_) in enumerate(p): candidate = template[name] # add variable to target if isinstance(candidate, StructureType): if name not in target.keys(): if i < len(p) - 1: # if there are more children to add we need to clear # the candidate so it has only explicitly added # children; also, Grids are degenerated into Structures if isinstance(candidate, GridType): candidate = StructureType( candidate.name, candidate.attributes) candidate._keys = [] target[name] = candidate target, template = target[name], template[name] else: target[name] = candidate # fix sequence data to include only variables that are in the sequence for seq in walk(out, SequenceType): seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data # apply slices for p in projection: target = out for name, slice_ in p: target, parent = target[name], target if slice_: if isinstance(target, BaseType): target.data = target[slice_] elif isinstance(target, SequenceType): parent[name] = target[slice_[0]] elif isinstance(target, GridType): parent[name] = target[slice_] else: raise ConstraintExpressionError("Invalid projection!") return out
def apply_projection(projection, dataset): """ Apply a given projection to a dataset. The function returns a new dataset object, after applying the projection to the original dataset. """ out = DatasetType(name=dataset.name, attributes=dataset.attributes) for var in projection: target, template = out, dataset while var: name, slice_ = var.pop(0) candidate = template[name] # apply slice if slice_: if isinstance(candidate, BaseType): candidate.data = candidate[slice_] elif isinstance(candidate, SequenceType): candidate = candidate[slice_[0]] elif isinstance(candidate, GridType): candidate = candidate[slice_] # handle structures if isinstance(candidate, StructureType): # add variable to target if name not in target.keys(): if var: # if there are more children to add we need to clear the # candidate so it has only explicitly added children; # also, Grids are degenerated into Structures if isinstance(candidate, GridType): candidate = StructureType(candidate.name, candidate.attributes) candidate._keys = [] target[name] = candidate target, template = target[name], template[name] else: target[name] = candidate # fix sequence data, including only variables that are in the sequence for seq in walk(out, SequenceType): seq.data = get_var(dataset, seq.id)[tuple(seq.keys())].data return out