Example #1
0
 def shape(self):
     shape_realization = get_default_or_apply(self.realization,len,1)
     shape_temporal = get_default_or_apply(self.temporal,len,1)
     shape_level = get_default_or_apply(self.level,len,1)
     shape_spatial = get_default_or_apply(self.spatial,lambda x: x.shape,(1,1))
     ret = (shape_realization,shape_temporal,shape_level,shape_spatial[0],shape_spatial[1])
     return(ret)
Example #2
0
    def shape(self):
        """
        :returns: The shape of the field as a five-element tuple: (realization, time, level, row, column)
        :rtype: tuple
        """

        shape_realization = get_default_or_apply(self.realization, len, 1)
        shape_temporal = get_default_or_apply(self.temporal, len, 1)
        shape_level = get_default_or_apply(self.level, len, 1)
        shape_spatial = get_default_or_apply(self.spatial, lambda x: x.shape, (1, 1))
        ret = (shape_realization, shape_temporal, shape_level, shape_spatial[0], shape_spatial[1])
        return ret
Example #3
0
    def __init__(self,
                 alias=None,
                 dtype=None,
                 field=None,
                 file_only=False,
                 vc=None,
                 parms=None,
                 tgd=None,
                 calc_sample_size=False,
                 fill_value=None,
                 meta_attrs=None,
                 tag=TagName.DATA_VARIABLES,
                 spatial_aggregation=False):

        self._curr_variable = None
        self._current_conformed_array = None
        self._dtype = dtype

        self.alias = alias or self.key
        self.fill_value = fill_value
        self.vc = vc or VariableCollection()
        self.field = field
        self.file_only = file_only
        self.parms = get_default_or_apply(parms,
                                          self._format_parms_,
                                          default={})
        self.tgd = tgd
        self.calc_sample_size = calc_sample_size
        self.meta_attrs = deepcopy(meta_attrs)
        self.tag = tag
        self.spatial_aggregation = spatial_aggregation
Example #4
0
    def __init__(
        self,
        alias=None,
        dtype=None,
        field=None,
        file_only=False,
        vc=None,
        parms=None,
        tgd=None,
        use_raw_values=False,
        calc_sample_size=False,
        fill_value=None,
        meta_attrs=None,
        add_parents=False,
    ):

        self._dtype = self.get_dtype(overload=dtype)
        self.alias = alias or self.key
        self.fill_value = fill_value
        self.vc = vc or VariableCollection()
        self.field = field
        self.file_only = file_only
        self.parms = get_default_or_apply(parms, self._format_parms_, default={})
        self.tgd = tgd
        self.use_raw_values = use_raw_values
        self.calc_sample_size = calc_sample_size
        self.meta_attrs = deepcopy(meta_attrs)
        self.add_parents = add_parents
Example #5
0
 def __init__(self,alias=None,dtype=None,field=None,file_only=False,vc=None,
              parms=None,tgd=None,use_raw_values=False,calc_sample_size=False,
              fill_value=None):
     self.alias = alias or self.key
     self.dtype = dtype or self.dtype
     self.fill_value = fill_value
     self.vc = vc or VariableCollection()
     self.field = field
     self.file_only = file_only
     self.parms = get_default_or_apply(parms,self._format_parms_,default={})
     self.tgd = tgd
     self.use_raw_values = use_raw_values
     self.calc_sample_size = calc_sample_size
Example #6
0
File: base.py Project: NCPP/ocgis
    def __init__(self, alias=None, dtype=None, field=None, file_only=False, vc=None, parms=None, tgd=None,
                 calc_sample_size=False, fill_value=None, meta_attrs=None, tag=TagName.DATA_VARIABLES,
                 spatial_aggregation=False):

        self._curr_variable = None
        self._current_conformed_array = None
        self._dtype = dtype

        self.alias = alias or self.key
        self.fill_value = fill_value
        self.vc = vc or VariableCollection()
        self.field = field
        self.file_only = file_only
        self.parms = get_default_or_apply(parms, self._format_parms_, default={})
        self.tgd = tgd
        self.calc_sample_size = calc_sample_size
        self.meta_attrs = deepcopy(meta_attrs)
        self.tag = tag
        self.spatial_aggregation = spatial_aggregation
Example #7
0
    def _iter_collections_(self):
        '''
        :yields: :class:`~ocgis.SpatialCollection`
        '''
        
        ## multivariate calculations require datasets come in as a list with all
        ## variable inputs part of the same sequence.
        if self._has_multivariate_calculations:
            itr_rd = [[r for r in self.ops.dataset.itervalues()]]

        ## otherwise, process geometries expects a single element sequence
        else:
            itr_rd = [[rd] for rd in self.ops.dataset.itervalues()]
        
        ## configure the progress object
        self._progress.n_subsettables = len(itr_rd)
        self._progress.n_geometries = get_default_or_apply(self.ops.geom,len,default=1)
        self._progress.n_calculations = get_default_or_apply(self.ops.calc,len,default=0)
        ## send some messages
        msg = '{0} dataset collection(s) to process.'.format(self._progress.n_subsettables)
        ocgis_lh(msg=msg,logger=self._subset_log)
        if self.ops.geom is None:
            msg = 'Entire spatial domain returned. No selection geometries requested.'
        else:
            msg = 'Each data collection will be subsetted by {0} selection geometries.'.format(self._progress.n_geometries)
        ocgis_lh(msg=msg,logger=self._subset_log)
        if self._progress.n_calculations == 0:
            msg = 'No calculations requested.'
        else:
            msg = 'The following calculations will be applied to each data collection: {0}.'.\
             format(', '.join([_['func'] for _ in self.ops.calc]))
        ocgis_lh(msg=msg,logger=self._subset_log)
        
        ## process the data collections
        for rds in itr_rd:
            msg = 'Processing URI(s): {0}'.format([rd.uri for rd in rds])
            ocgis_lh(msg=msg,logger=self._subset_log)
            
            for coll in self._process_subsettables_(rds):
                ## if there are calculations, do those now and return a new type of collection
                if self.cengine is not None:
                    ocgis_lh('Starting calculations.',
                             self._subset_log,
                             alias=coll.items()[0][1].keys()[0],
                             ugid=coll.keys()[0])
                    
                    ## look for any optimizations for temporal grouping.
                    if self.ops.optimizations is None:
                        tgds = None
                    else:
                        tgds = self.ops.optimizations.get('tgds')
                    ## execute the calculations
                    coll = self.cengine.execute(coll,file_only=self.ops.file_only,
                                                tgds=tgds)
                else:
                    ## if there are no calculations, mark progress to indicate
                    ## a geometry has been completed.
                    self._progress.mark()
                
                ## conversion of groups.
                if self.ops.output_grouping is not None:
                    raise(NotImplementedError)
                else:
                    ocgis_lh('subset yielding',self._subset_log,level=logging.DEBUG)
                    yield(coll)
Example #8
0
    def _iter_collections_(self):
        """:rtype: :class:`ocgis.collection.base.AbstractCollection`"""

        # Multivariate calculations require datasets come in as a list with all variable inputs part of the same
        # sequence.
        if self._has_multivariate_calculations:
            itr_rd = [[rd for rd in self.ops.dataset]]
        # Otherwise, process geometries expects a single element sequence.
        else:
            itr_rd = [[rd] for rd in self.ops.dataset]

        # Configure the progress object.
        self._progress.n_subsettables = len(itr_rd)
        self._progress.n_geometries = get_default_or_apply(self.ops.geom,
                                                           len,
                                                           default=1)
        self._progress.n_calculations = get_default_or_apply(self.ops.calc,
                                                             len,
                                                             default=0)

        # Some introductory logging.
        msg = '{0} dataset collection(s) to process.'.format(
            self._progress.n_subsettables)
        ocgis_lh(msg=msg, logger=self._subset_log)
        if self.ops.geom is None:
            msg = 'Entire spatial domain returned. No selection geometries requested.'
        else:
            msg = 'Each data collection will be subsetted by {0} selection geometries.'.format(
                self._progress.n_geometries)
        ocgis_lh(msg=msg, logger=self._subset_log)
        if self._progress.n_calculations == 0:
            msg = 'No calculations requested.'
        else:
            msg = 'The following calculations will be applied to each data collection: {0}.'. \
                format(', '.join([_['func'] for _ in self.ops.calc]))
        ocgis_lh(msg=msg, logger=self._subset_log)

        # Process the incoming datasets. Convert from request datasets to fields as needed.
        for rds in itr_rd:

            try:
                msg = 'Processing URI(s): {0}'.format([rd.uri for rd in rds])
            except AttributeError:
                # Field objects have no URIs. Multivariate calculations change how the request dataset iterator is
                # configured as well.
                msg = []
                for rd in rds:
                    try:
                        msg.append(rd.uri)
                    except AttributeError:
                        # Likely a field object which does have a name.
                        msg.append(rd.name)
                msg = 'Processing URI(s) / field names: {0}'.format(msg)
            ocgis_lh(msg=msg, logger=self._subset_log)

            for coll in self._process_subsettables_(rds):
                # If there are calculations, do those now and return a collection.
                if not vm.is_null and self.cengine is not None:
                    ocgis_lh('Starting calculations.', self._subset_log)
                    raise_if_empty(coll)

                    # Look for any temporal grouping optimizations.
                    if self.ops.optimizations is None:
                        tgds = None
                    else:
                        tgds = self.ops.optimizations.get('tgds')

                    # Execute the calculations.
                    coll = self.cengine.execute(coll,
                                                file_only=self.ops.file_only,
                                                tgds=tgds)

                    # If we need to spatially aggregate and calculations used raw values, update the collection
                    # fields and subset geometries.
                    if self.ops.aggregate and self.ops.calc_raw:
                        coll_to_itr = coll.copy()
                        for sfield, container in coll_to_itr.iter_fields(
                                yield_container=True):
                            sfield = _update_aggregation_wrapping_crs_(
                                self, None, sfield, container, None)
                            coll.add_field(sfield, container, force=True)
                else:
                    # If there are no calculations, mark progress to indicate a geometry has been completed.
                    self._progress.mark()

                # Conversion of groups.
                if self.ops.output_grouping is not None:
                    raise NotImplementedError
                else:
                    ocgis_lh('_iter_collections_ yielding',
                             self._subset_log,
                             level=logging.DEBUG)
                    yield coll
Example #9
0
File: engine.py Project: NCPP/ocgis
    def _iter_collections_(self):
        """:rtype: :class:`ocgis.collection.base.AbstractCollection`"""

        # Multivariate calculations require datasets come in as a list with all variable inputs part of the same
        # sequence.
        if self._has_multivariate_calculations:
            itr_rd = [[rd for rd in self.ops.dataset]]
        # Otherwise, process geometries expects a single element sequence.
        else:
            itr_rd = [[rd] for rd in self.ops.dataset]

        # Configure the progress object.
        self._progress.n_subsettables = len(itr_rd)
        self._progress.n_geometries = get_default_or_apply(self.ops.geom, len, default=1)
        self._progress.n_calculations = get_default_or_apply(self.ops.calc, len, default=0)

        # Some introductory logging.
        msg = '{0} dataset collection(s) to process.'.format(self._progress.n_subsettables)
        ocgis_lh(msg=msg, logger=self._subset_log)
        if self.ops.geom is None:
            msg = 'Entire spatial domain returned. No selection geometries requested.'
        else:
            msg = 'Each data collection will be subsetted by {0} selection geometries.'.format(
                self._progress.n_geometries)
        ocgis_lh(msg=msg, logger=self._subset_log)
        if self._progress.n_calculations == 0:
            msg = 'No calculations requested.'
        else:
            msg = 'The following calculations will be applied to each data collection: {0}.'. \
                format(', '.join([_['func'] for _ in self.ops.calc]))
        ocgis_lh(msg=msg, logger=self._subset_log)

        # Process the incoming datasets. Convert from request datasets to fields as needed.
        for rds in itr_rd:

            try:
                msg = 'Processing URI(s): {0}'.format([rd.uri for rd in rds])
            except AttributeError:
                # Field objects have no URIs. Multivariate calculations change how the request dataset iterator is
                # configured as well.
                msg = []
                for rd in rds:
                    try:
                        msg.append(rd.uri)
                    except AttributeError:
                        # Likely a field object which does have a name.
                        msg.append(rd.name)
                msg = 'Processing URI(s) / field names: {0}'.format(msg)
            ocgis_lh(msg=msg, logger=self._subset_log)

            for coll in self._process_subsettables_(rds):
                # If there are calculations, do those now and return a collection.
                if not vm.is_null and self.cengine is not None:
                    ocgis_lh('Starting calculations.', self._subset_log)
                    raise_if_empty(coll)

                    # Look for any temporal grouping optimizations.
                    if self.ops.optimizations is None:
                        tgds = None
                    else:
                        tgds = self.ops.optimizations.get('tgds')

                    # Execute the calculations.
                    coll = self.cengine.execute(coll, file_only=self.ops.file_only, tgds=tgds)

                    # If we need to spatially aggregate and calculations used raw values, update the collection
                    # fields and subset geometries.
                    if self.ops.aggregate and self.ops.calc_raw:
                        coll_to_itr = coll.copy()
                        for sfield, container in coll_to_itr.iter_fields(yield_container=True):
                            sfield = _update_aggregation_wrapping_crs_(self, None, sfield, container, None)
                            coll.add_field(sfield, container, force=True)
                else:
                    # If there are no calculations, mark progress to indicate a geometry has been completed.
                    self._progress.mark()

                # Conversion of groups.
                if self.ops.output_grouping is not None:
                    raise NotImplementedError
                else:
                    ocgis_lh('_iter_collections_ yielding', self._subset_log, level=logging.DEBUG)
                    yield coll