Esempio n. 1
0
 def test_add_variable_already_in_collection_uids_update(self):
     vc = VariableCollection()
     var = self.get_variable()
     vc.add_variable(var)
     self.assertEqual(var.uid, 1)
     var.alias = 'variable_2'
     vc.add_variable(var, assign_new_uid=True)
     self.assertEqual(var.uid, 2)
     self.assertEqual(vc._storage_id, [1, 2])
Esempio n. 2
0
    def _get_field_(self, format_time=None):
        # todo: option to pass select_ugid
        # todo: option for time dimension and time subsetting
        # todo: remove format_time option - there for compatibility with the netCDF driver
        from ocgis import SpatialDimension

        ds = self.open()
        try:
            records = list(ds)
            sdim = SpatialDimension.from_records(records, crs=self.get_crs())
            # do not load the properties - they are transformed to variables in the case of the values put into fields
            sdim.properties = None
            vc = VariableCollection()
            for xx in self.rd:
                value = np.array([yy['properties'][xx['variable']] for yy in records]).reshape(1, 1, 1, 1, -1)
                var = Variable(name=xx['variable'], alias=xx['alias'], units=xx['units'], conform_units_to=xx['units'],
                               value=value)
                vc.add_variable(var, assign_new_uid=True)
            field = Field(spatial=sdim, variables=vc, name=self.rd.name)
            return field
        finally:
            self.close(ds)
Esempio n. 3
0
 def test_add_variable(self):
     vc = VariableCollection()
     var = self.get_variable()
     self.assertEqual(var.uid, None)
     vc.add_variable(var)
     self.assertEqual(var.uid, 1)
     self.assertTrue('tas_foo' in vc)
     self.assertEqual(vc._storage_id, [1])
     var.alias = 'again'
     with self.assertRaises(AssertionError):
         vc.add_variable(var)
     var.uid = 100
     vc.add_variable(var)
     self.assertEqual(vc._storage_id, [1, 100])
Esempio n. 4
0
    def _get_field_(self, format_time=True):
        """
        :param bool format_time:
        :raises ValueError:
        """

        # reference the request dataset's source metadata
        source_metadata = self.rd.source_metadata

        def _get_temporal_adds_(ref_attrs):
            # calendar should default to standard if it is not present and the t_calendar overload is not used.
            calendar = self.rd.t_calendar or ref_attrs.get("calendar", None) or "standard"

            return {
                "units": self.rd.t_units or ref_attrs["units"],
                "calendar": calendar,
                "format_time": format_time,
                "conform_units_to": self.rd.t_conform_units_to,
            }

        # parameters for the loading loop
        to_load = {
            "temporal": {
                "cls": NcTemporalDimension,
                "adds": _get_temporal_adds_,
                "axis": "T",
                "name_uid": "tid",
                "name": "time",
            },
            "level": {"cls": NcVectorDimension, "adds": None, "axis": "Z", "name_uid": "lid", "name": "level"},
            "row": {"cls": NcVectorDimension, "adds": None, "axis": "Y", "name_uid": "yc_id", "name": "yc"},
            "col": {"cls": NcVectorDimension, "adds": None, "axis": "X", "name_uid": "xc_id", "name": "xc"},
            "realization": {
                "cls": NcVectorDimension,
                "adds": None,
                "axis": "R",
                "name_uid": "rlz_id",
                "name_value": "rlz",
            },
        }

        loaded = {}
        kwds_grid = {}
        has_row_column = True
        for k, v in to_load.iteritems():
            fill = self._get_vector_dimension_(k, v, source_metadata)
            if k != "realization" and not isinstance(fill, NcVectorDimension) and fill is not None:
                assert k in ("row", "col")
                has_row_column = False
                kwds_grid[k] = fill
            loaded[k] = fill

        loaded_keys = set([k for k, v in loaded.iteritems() if v is not None])
        if has_row_column:
            if not {"temporal", "row", "col"}.issubset(loaded_keys):
                raise ValueError("Target variable must at least have temporal, row, and column dimensions.")
            kwds_grid = {"row": loaded["row"], "col": loaded["col"]}
        else:
            shape_src_idx = [source_metadata["dimensions"][xx]["len"] for xx in kwds_grid["row"]["dimensions"]]
            src_idx = {
                "row": np.arange(0, shape_src_idx[0], dtype=np.int32),
                "col": np.arange(0, shape_src_idx[1], dtype=np.int32),
            }
            name_row = kwds_grid["row"]["name"]
            name_col = kwds_grid["col"]["name"]
            kwds_grid = {"name_row": name_row, "name_col": name_col, "data": self.rd, "src_idx": src_idx}

        grid = NcSpatialGridDimension(**kwds_grid)

        spatial = SpatialDimension(name_uid="gid", grid=grid, crs=self.rd.crs, abstraction=self.rd.s_abstraction)

        vc = VariableCollection()
        for vdict in self.rd:
            variable_meta = deepcopy(source_metadata["variables"][vdict["variable"]])
            variable_units = vdict["units"] or variable_meta["attrs"].get("units")
            dtype = np.dtype(variable_meta["dtype"])
            fill_value = variable_meta["fill_value"]
            variable = Variable(
                vdict["variable"],
                vdict["alias"],
                units=variable_units,
                meta=variable_meta,
                data=self.rd,
                conform_units_to=vdict["conform_units_to"],
                dtype=dtype,
                fill_value=fill_value,
                attrs=variable_meta["attrs"].copy(),
            )
            vc.add_variable(variable)

        ret = NcField(
            variables=vc,
            spatial=spatial,
            temporal=loaded["temporal"],
            level=loaded["level"],
            realization=loaded["realization"],
            meta=source_metadata.copy(),
            uid=self.rd.did,
            name=self.rd.name,
            attrs=source_metadata["dataset"].copy(),
        )

        # Apply any subset parameters after the field is loaded.
        if self.rd.time_range is not None:
            ret = ret.get_between("temporal", min(self.rd.time_range), max(self.rd.time_range))
        if self.rd.time_region is not None:
            ret = ret.get_time_region(self.rd.time_region)
        if self.rd.time_subset_func is not None:
            ret = ret.get_time_subset_by_function(self.rd.time_subset_func)
        if self.rd.level_range is not None:
            try:
                ret = ret.get_between("level", min(self.rd.level_range), max(self.rd.level_range))
            except AttributeError:
                # there may be no level dimension
                if ret.level is None:
                    msg = messages.M4.format(self.rd.alias)
                    raise ValueError(msg)
                else:
                    raise

        return ret
Esempio n. 5
0
    def get_field(self, format_time=True, interpolate_spatial_bounds=False):
        """
        :param bool format_time:
        :param bool interpolate_spatial_bounds:
        :raises ValueError:
        """

        def _get_temporal_adds_(ref_attrs):
            ## calendar should default to standard if it is not present and the
            ## t_calendar overload is not used.
            calendar = self.rd.t_calendar or ref_attrs.get('calendar', None) or 'standard'

            return ({'units': self.rd.t_units or ref_attrs['units'],
                     'calendar': calendar,
                     'format_time': format_time})

        ## this dictionary contains additional keyword arguments for the row
        ## and column dimensions.
        adds_row_col = {'interpolate_bounds': interpolate_spatial_bounds}

        ## parameters for the loading loop
        to_load = {'temporal': {'cls': NcTemporalDimension, 'adds': _get_temporal_adds_, 'axis': 'T', 'name_uid': 'tid',
                                'name_value': 'time'},
                   'level': {'cls': NcVectorDimension, 'adds': None, 'axis': 'Z', 'name_uid': 'lid',
                             'name_value': 'level'},
                   'row': {'cls': NcVectorDimension, 'adds': adds_row_col, 'axis': 'Y', 'name_uid': 'row_id',
                           'name_value': 'row'},
                   'col': {'cls': NcVectorDimension, 'adds': adds_row_col, 'axis': 'X', 'name_uid': 'col_id',
                           'name_value': 'col'},
                   'realization': {'cls': NcVectorDimension, 'adds': None, 'axis': 'R', 'name_uid': 'rlz_id',
                                   'name_value': 'rlz'}}
        loaded = {}

        for k, v in to_load.iteritems():
            ## this is the string axis representation
            axis_value = v['axis'] or v['cls']._axis
            ## pull the axis information out of the dimension map
            ref_axis = self.rd.source_metadata['dim_map'].get(axis_value)
            ref_axis = self.rd.source_metadata['dim_map'].get(axis_value)
            ## if the axis is not represented, fill it with none. this happens
            ## when a dataset does not have a vertical level or projection axis
            ## for example.
            if ref_axis is None:
                fill = None
            else:
                ref_variable = self.rd.source_metadata['variables'].get(ref_axis['variable'])

                ## for data with a projection/realization axis there may be no
                ## associated variable.
                try:
                    ref_variable['axis'] = ref_axis
                except TypeError:
                    if axis_value == 'R' and ref_variable is None:
                        ref_variable = {'axis': ref_axis, 'name': ref_axis['dimension'], 'attrs': {}}

                ## extract the data length to use when creating the source index
                ## arrays.
                length = self.rd.source_metadata['dimensions'][ref_axis['dimension']]['len']
                src_idx = np.arange(0, length, dtype=constants.np_int)

                ## get the target data type for the dimension
                try:
                    dtype = np.dtype(ref_variable['dtype'])
                ## the realization dimension may not be a associated with a variable
                except KeyError:
                    if k == 'realization' and ref_variable['axis']['variable'] is None:
                        dtype = None
                    else:
                        raise

                ## assemble parameters for creating the dimension class then initialize
                ## the class.
                kwds = dict(name_uid=v['name_uid'], name_value=v['name_value'], src_idx=src_idx,
                            data=self.rd, meta=ref_variable, axis=axis_value, name=ref_variable.get('name'),
                            dtype=dtype)

                ## there may be additional parameters for each dimension.
                if v['adds'] is not None:
                    try:
                        kwds.update(v['adds'](ref_variable['attrs']))
                    ## adds may not be a callable object. assume they are a
                    ## dictionary.
                    except TypeError:
                        kwds.update(v['adds'])
                kwds.update({'name': ref_variable.get('name')})
                fill = v['cls'](**kwds)

            loaded[k] = fill

        assert_raise(set(('temporal', 'row', 'col')).issubset(set([k for k, v in loaded.iteritems() if v != None])),
                     logger='request',
                     exc=ValueError('Target variable must at least have temporal, row, and column dimensions.'))

        grid = SpatialGridDimension(row=loaded['row'], col=loaded['col'])

        # crs = None
        # if rd.crs is not None:
        #     crs = rd.crs
        # else:
        #     crs = rd._get_crs_(rd._variable[0])
        # if crs is None:
        #     ocgis_lh('No "grid_mapping" attribute available assuming WGS84: {0}'.format(rd.uri),
        #              'request', logging.WARN)
        #     crs = CFWGS84()

        spatial = SpatialDimension(name_uid='gid', grid=grid, crs=self.rd.crs, abstraction=self.rd.s_abstraction)

        vc = VariableCollection()
        for vdict in self.rd:
            variable_meta = deepcopy(self.rd._source_metadata['variables'][vdict['variable']])
            variable_units = vdict['units'] or variable_meta['attrs'].get('units')
            dtype = np.dtype(variable_meta['dtype'])
            fill_value = variable_meta['fill_value']
            variable = Variable(vdict['variable'], vdict['alias'], units=variable_units, meta=variable_meta,
                                data=self.rd, conform_units_to=vdict['conform_units_to'], dtype=dtype,
                                fill_value=fill_value)
            vc.add_variable(variable)

        ret = NcField(variables=vc, spatial=spatial, temporal=loaded['temporal'], level=loaded['level'],
                      realization=loaded['realization'], meta=deepcopy(self.rd._source_metadata), uid=self.rd.did,
                      name=self.rd.name)

        ## apply any subset parameters after the field is loaded
        if self.rd.time_range is not None:
            ret = ret.get_between('temporal', min(self.rd.time_range), max(self.rd.time_range))
        if self.rd.time_region is not None:
            ret = ret.get_time_region(self.rd.time_region)
        if self.rd.level_range is not None:
            try:
                ret = ret.get_between('level', min(self.rd.level_range), max(self.rd.level_range))
            except AttributeError:
                ## there may be no level dimension
                if ret.level == None:
                    msg = ("A level subset was requested but the target dataset does not have a level dimension. The "
                           "dataset's alias is: {0}".format(self.rd.alias))
                    raise (ValueError(msg))
                else:
                    raise

        return ret
Esempio n. 6
0
 def test_add_variable_already_in_collection(self):
     vc = VariableCollection()
     var = self.get_variable()
     vc.add_variable(var)
     with self.assertRaises(VariableInCollectionError):
         vc.add_variable(var)
Esempio n. 7
0
    def _get_field_(self, format_time=True):
        """
        :param bool format_time:
        :raises ValueError:
        """

        # reference the request dataset's source metadata
        source_metadata = self.rd.source_metadata

        def _get_temporal_adds_(ref_attrs):
            # calendar should default to standard if it is not present and the t_calendar overload is not used.
            calendar = self.rd.t_calendar or ref_attrs.get('calendar', None) or 'standard'

            return {'units': self.rd.t_units or ref_attrs['units'], 'calendar': calendar, 'format_time': format_time,
                    'conform_units_to': self.rd.t_conform_units_to}

        # parameters for the loading loop
        to_load = {'temporal': {'cls': NcTemporalDimension, 'adds': _get_temporal_adds_, 'axis': 'T', 'name_uid': 'tid',
                                'name': 'time'},
                   'level': {'cls': NcVectorDimension, 'adds': None, 'axis': 'Z', 'name_uid': 'lid', 'name': 'level'},
                   'row': {'cls': NcVectorDimension, 'adds': None, 'axis': 'Y', 'name_uid': 'yc_id', 'name': 'yc'},
                   'col': {'cls': NcVectorDimension, 'adds': None, 'axis': 'X', 'name_uid': 'xc_id', 'name': 'xc'},
                   'realization': {'cls': NcVectorDimension, 'adds': None, 'axis': 'R', 'name_uid': 'rlz_id',
                                   'name_value': 'rlz'}}

        loaded = {}
        kwds_grid = {}
        has_row_column = True
        for k, v in to_load.iteritems():
            fill = self._get_vector_dimension_(k, v, source_metadata)
            if k != 'realization' and not isinstance(fill, NcVectorDimension) and fill is not None:
                assert k in ('row', 'col')
                has_row_column = False
                kwds_grid[k] = fill
            loaded[k] = fill

        loaded_keys = set([k for k, v in loaded.iteritems() if v is not None])
        if has_row_column:
            if not {'temporal', 'row', 'col'}.issubset(loaded_keys):
                raise ValueError('Target variable must at least have temporal, row, and column dimensions.')
            kwds_grid = {'row': loaded['row'], 'col': loaded['col']}
        else:
            shape_src_idx = [source_metadata['dimensions'][xx]['len'] for xx in kwds_grid['row']['dimensions']]
            src_idx = {'row': np.arange(0, shape_src_idx[0], dtype=np.int32),
                       'col': np.arange(0, shape_src_idx[1], dtype=np.int32)}
            name_row = kwds_grid['row']['name']
            name_col = kwds_grid['col']['name']
            kwds_grid = {'name_row': name_row, 'name_col': name_col, 'request_dataset': self.rd, 'src_idx': src_idx}

        grid = NcSpatialGridDimension(**kwds_grid)

        spatial = SpatialDimension(name_uid='gid', grid=grid, crs=self.rd.crs, abstraction=self.rd.s_abstraction)

        vc = VariableCollection()
        for vdict in self.rd:
            variable_meta = deepcopy(source_metadata['variables'][vdict['variable']])
            variable_units = vdict['units'] or variable_meta['attrs'].get('units')
            attrs = variable_meta['attrs'].copy()
            if variable_meta['dtype_packed'] is None:
                dtype = np.dtype(variable_meta['dtype'])
                fill_value = variable_meta['fill_value']
            else:
                dtype = np.dtype(variable_meta['dtype_packed'])
                fill_value = variable_meta['fill_value_packed']
                # Remove scale factors and offsets from the metadata.
                attrs.pop('scale_factor')
                attrs.pop('add_offset', None)
                attrs.pop('missing_value', None)
                attrs.pop('_Fill_Value', None)
            variable = Variable(vdict['variable'], vdict['alias'], units=variable_units, meta=variable_meta,
                                request_dataset=self.rd, conform_units_to=vdict['conform_units_to'], dtype=dtype,
                                fill_value=fill_value, attrs=attrs)
            vc.add_variable(variable)

        ret = NcField(variables=vc, spatial=spatial, temporal=loaded['temporal'], level=loaded['level'],
                      realization=loaded['realization'], meta=source_metadata.copy(), uid=self.rd.did,
                      name=self.rd.name, attrs=source_metadata['dataset'].copy())

        # Apply any subset parameters after the field is loaded.
        if self.rd.time_range is not None:
            ret = ret.get_between('temporal', min(self.rd.time_range), max(self.rd.time_range))
        if self.rd.time_region is not None:
            ret = ret.get_time_region(self.rd.time_region)
        if self.rd.time_subset_func is not None:
            ret = ret.get_time_subset_by_function(self.rd.time_subset_func)
        if self.rd.level_range is not None:
            try:
                ret = ret.get_between('level', min(self.rd.level_range), max(self.rd.level_range))
            except AttributeError:
                # there may be no level dimension
                if ret.level is None:
                    msg = messages.M4.format(self.rd.alias)
                    raise ValueError(msg)
                else:
                    raise

        return ret