def get_variable_metadata_from_request_dataset(driver, variable): variables_metadata = get_group(driver.metadata_source, variable.group, has_root=False)['variables'] try: ret = variables_metadata[variable._source_name] except KeyError: raise VariableMissingMetadataError(variable._source_name) return ret
def get_variable_metadata_from_request_dataset(driver, variable): variables_metadata = get_group(driver.metadata_source, variable.group, has_root=False)['variables'] try: ret = variables_metadata[variable._source_name] except KeyError: raise VariableMissingMetadataError(variable._source_name) return ret
def test_create_dist(self): def _create_dimensions_(ds, k): if k.dim_count > 0: ds.createDimension('one', 1) if k.dim_count == 2: ds.createDimension('two', 2) kwds = dict(dim_count=[0, 1, 2], nested=[False, True]) for k in self.iter_product_keywords(kwds): path = self.get_temporary_file_path('{}.nc'.format(k.dim_count)) with self.nc_scope(path, 'w') as ds: _create_dimensions_(ds, k) if k.nested: group1 = ds.createGroup('nest1') _create_dimensions_(group1, k) group2 = group1.createGroup('nest2') _create_dimensions_(group2, k) group3 = group2.createGroup('nest1') _create_dimensions_(group3, k) group3a = group2.createGroup('nest3') _create_dimensions_(group3a, k) group3.createDimension('outlier', 4) rd = RequestDataset(uri=path) driver = DriverNetcdf(rd) actual = driver.create_dist().mapping # All dimensions are not distributed. for keyseq in iter_all_group_keys(actual[MPI_RANK]): group = get_group(actual[MPI_RANK], keyseq) for dim in list(group['dimensions'].values()): self.assertFalse(dim.dist) if k.dim_count == 0 and k.nested: desired = {None: {'variables': {}, 'dimensions': {}, 'groups': { u'nest1': {'variables': {}, 'dimensions': {}, 'groups': { u'nest2': {'variables': {}, 'dimensions': {}, 'groups': {u'nest3': {'variables': {}, 'dimensions': {}, 'groups': {}}, u'nest1': {'variables': {}, 'dimensions': { u'outlier': Dimension(name='outlier', size=4, size_current=4, dist=False, is_empty=False, src_idx='auto')}, 'groups': {}}}}}}}}} self.assertEqual(actual[MPI_RANK], desired) if k.dim_count == 2 and k.nested: self.assertIsNotNone(driver.metadata_source['groups']['nest1']['groups']['nest2']) two_dimensions = [Dimension(name='one', size=1, size_current=1), Dimension(name='two', size=2, size_current=2)] nest1 = {'dimensions': two_dimensions, 'groups': {}} template = deepcopy(nest1) nest1['groups']['nest2'] = deepcopy(template) nest1['groups']['nest2']['groups']['nest1'] = deepcopy(template) nest1['groups']['nest2']['groups']['nest3'] = deepcopy(template) nest1['groups']['nest2']['groups']['nest1']['dimensions'].append(Dimension('outlier', 4)) desired = {None: {'dimensions': two_dimensions, 'groups': {'nest1': nest1}}} groups_actual = list(iter_all_group_keys((actual[MPI_RANK]))) groups_desired = list(iter_all_group_keys(desired)) self.assertEqual(groups_actual, groups_desired)
def create_dist(self, metadata=None): """ Create a distribution from global metadata. In general, this should not be overloaded by subclasses. :param dict metadata: Global metadata to use for creating a distribution. :rtype: :class:`ocgis.OcgDist` """ ompi = OcgDist(size=vm.size, ranks=vm.ranks) # Convert metadata into a grouping consistent with the MPI dimensions. if metadata is None: metadata = self.metadata_source metadata = {None: metadata} for group_index in iter_all_group_keys(metadata): group_meta = get_group(metadata, group_index) # Add the dimensions to the distribution object. dimensions = self.create_dimensions(group_meta) # Only build a distribution if the group has more than one dimension. if len(dimensions) == 0: _ = ompi.get_group(group=group_index) else: for dimension_name, dimension_meta in list( group_meta['dimensions'].items()): target_dimension = dimensions[dimension_name] target_dimension.dist = group_meta['dimensions'][ dimension_name].get('dist', False) ompi.add_dimension(target_dimension, group=group_index) try: dimension_map = self.rd.dimension_map.get_group( group_index) except DimensionMapError: # Likely a user-provided dimension map. continue # dimension_map = get_group(self.rd.dimension_map, group_index, has_root=False) distributed_dimension_name = self.get_distributed_dimension_name( dimension_map, group_meta['dimensions'], decomp_type=self.rd.decomp_type) # Allow no distributed dimensions to be returned. if distributed_dimension_name is not None: for target_rank in range(ompi.size): distributed_dimension = ompi.get_dimension( distributed_dimension_name, group=group_index, rank=target_rank) distributed_dimension.dist = True ompi.update_dimension_bounds() return ompi
def create_dist(self, metadata=None): """ Create a distribution from global metadata. In general, this should not be overloaded by subclasses. :param dict metadata: Global metadata to use for creating a distribution. :rtype: :class:`ocgis.OcgDist` """ ompi = OcgDist(size=vm.size, ranks=vm.ranks) # Convert metadata into a grouping consistent with the MPI dimensions. if metadata is None: metadata = self.metadata_source metadata = {None: metadata} for group_index in iter_all_group_keys(metadata): group_meta = get_group(metadata, group_index) # Add the dimensions to the distribution object. dimensions = self.create_dimensions(group_meta) # Only build a distribution if the group has more than one dimension. if len(dimensions) == 0: _ = ompi.get_group(group=group_index) else: for dimension_name, dimension_meta in list(group_meta['dimensions'].items()): target_dimension = dimensions[dimension_name] target_dimension.dist = group_meta['dimensions'][dimension_name].get('dist', False) ompi.add_dimension(target_dimension, group=group_index) try: dimension_map = self.rd.dimension_map.get_group(group_index) except DimensionMapError: # Likely a user-provided dimension map. continue # dimension_map = get_group(self.rd.dimension_map, group_index, has_root=False) distributed_dimension_name = self.get_distributed_dimension_name(dimension_map, group_meta['dimensions'], decomp_type=self.rd.decomp_type) # Allow no distributed dimensions to be returned. if distributed_dimension_name is not None: for target_rank in range(ompi.size): distributed_dimension = ompi.get_dimension(distributed_dimension_name, group=group_index, rank=target_rank) distributed_dimension.dist = True ompi.update_dimension_bounds() return ompi
def get_dist(self): """ :rtype: :class:`ocgis.OcgVM` """ ompi = OcgDist(size=vm.size, ranks=vm.ranks) # ompi = OcgDist() # Convert metadata into a grouping consistent with the MPI dimensions. metadata = {None: self.metadata_source} for group_index in iter_all_group_keys(metadata): group_meta = get_group(metadata, group_index) # Add the dimensions to the distribution object. dimensions = self._get_dimensions_main_(group_meta) for dimension_name, dimension_meta in list(group_meta['dimensions'].items()): target_dimension = find_dimension_in_sequence(dimension_name, dimensions) target_dimension.dist = group_meta['dimensions'][dimension_name].get('dist', False) ompi.add_dimension(target_dimension, group=group_index) try: dimension_map = self.rd.dimension_map.get_group(group_index) except DimensionMapError: # Likely a user-provided dimension map. continue # dimension_map = get_group(self.rd.dimension_map, group_index, has_root=False) distributed_dimension_name = self.get_distributed_dimension_name(dimension_map, group_meta['dimensions']) # Allow no distributed dimensions to be returned. if distributed_dimension_name is not None: for target_rank in range(ompi.size): distributed_dimension = ompi.get_dimension(distributed_dimension_name, group=group_index, rank=target_rank) distributed_dimension.dist = True # Add the variables to the distribution object. for variable_name, variable_meta in list(group_meta['variables'].items()): ompi.add_variable(variable_name, dimensions=variable_meta['dimensions'], group=group_index) # tdk: this will have to be moved to account for slicing ompi.update_dimension_bounds() return ompi
def init_variable_using_metadata_for_netcdf(variable, metadata): source = get_group(metadata, variable.group, has_root=False) desired_name = variable.source_name var = source['variables'][desired_name] if vm.is_null: variable.convert_to_empty() else: # Update data type and fill value. if is_auto_dtype( variable._dtype) or var.get('dtype_packed') is not None: var_dtype = var['dtype'] desired_dtype = deepcopy(var_dtype) if isinstance(var_dtype, VLType): desired_dtype = ObjectType(var_dtype) elif var['dtype_packed'] is not None: desired_dtype = deepcopy(var['dtype_packed']) variable._dtype = desired_dtype if variable._fill_value == 'auto': if var.get('fill_value_packed') is not None: desired_fill_value = var['fill_value_packed'] else: desired_fill_value = var.get('fill_value') variable._fill_value = deepcopy(desired_fill_value) variable_attrs = variable._attrs # Offset and scale factors are not supported by OCGIS. The data is unpacked when written to a new output file. # TODO: Consider supporting offset and scale factors for write operations. exclude = ['add_offset', 'scale_factor'] for k, v in list(var.get('attrs', {}).items()): if k in exclude: continue if k not in variable_attrs: variable_attrs[k] = deepcopy(v) # The conform units to value should be the default units value. Units will be converted on variable load. conform_units_to = var.get('conform_units_to') if conform_units_to is not None: variable_attrs['units'] = conform_units_to
def init_variable_using_metadata_for_netcdf(variable, metadata): source = get_group(metadata, variable.group, has_root=False) desired_name = variable.source_name var = source['variables'][desired_name] if vm.is_null: variable.convert_to_empty() else: # Update data type and fill value. if is_auto_dtype(variable._dtype) or var.get('dtype_packed') is not None: var_dtype = var['dtype'] desired_dtype = deepcopy(var_dtype) if isinstance(var_dtype, VLType): desired_dtype = ObjectType(var_dtype) elif var['dtype_packed'] is not None: desired_dtype = deepcopy(var['dtype_packed']) variable._dtype = desired_dtype if variable._fill_value == 'auto': if var.get('fill_value_packed') is not None: desired_fill_value = var['fill_value_packed'] else: desired_fill_value = var.get('fill_value') variable._fill_value = deepcopy(desired_fill_value) variable_attrs = variable._attrs # Offset and scale factors are not supported by OCGIS. The data is unpacked when written to a new output file. # TODO: Consider supporting offset and scale factors for write operations. exclude = ['add_offset', 'scale_factor'] for k, v in list(var.get('attrs', {}).items()): if k in exclude: continue if k not in variable_attrs: variable_attrs[k] = deepcopy(v) # The conform units to value should be the default units value. Units will be converted on variable load. conform_units_to = var.get('conform_units_to') if conform_units_to is not None: variable_attrs['units'] = conform_units_to
def iter_group_keys(ddict, keyseq): for key in get_group(ddict, keyseq).get('groups', {}): yld = deepcopy(keyseq) yld.append(key) yield yld
def get_group_metadata(group_index, metadata, has_root=False): return get_group(metadata, group_index, has_root=has_root)
def get_group_metadata(group_index, metadata, has_root=False): return get_group(metadata, group_index, has_root=has_root)
def iter_group_keys(ddict, keyseq): for key in get_group(ddict, keyseq).get('groups', {}): yld = deepcopy(keyseq) yld.append(key) yield yld
def iter_groups(self, rank=MPI_RANK): from ocgis.driver.base import iter_all_group_keys mapping = self.mapping[rank] for group_key in iter_all_group_keys(mapping): group_data = get_group(mapping, group_key) yield group_key, group_data
def test_get_dist(self): def _create_dimensions_(ds, k): if k.dim_count > 0: ds.createDimension('one', 1) if k.dim_count == 2: ds.createDimension('two', 2) kwds = dict(dim_count=[0, 1, 2], nested=[False, True]) for k in self.iter_product_keywords(kwds): path = self.get_temporary_file_path('{}.nc'.format(k.dim_count)) with self.nc_scope(path, 'w') as ds: _create_dimensions_(ds, k) if k.nested: group1 = ds.createGroup('nest1') _create_dimensions_(group1, k) group2 = group1.createGroup('nest2') _create_dimensions_(group2, k) group3 = group2.createGroup('nest1') _create_dimensions_(group3, k) group3a = group2.createGroup('nest3') _create_dimensions_(group3a, k) group3.createDimension('outlier', 4) rd = RequestDataset(uri=path) driver = DriverNetcdf(rd) actual = driver.get_dist().mapping # All dimensions are not distributed. for keyseq in iter_all_group_keys(actual[MPI_RANK]): group = get_group(actual[MPI_RANK], keyseq) for dim in list(group['dimensions'].values()): self.assertFalse(dim.dist) if k.dim_count == 0 and k.nested: desired = { None: { 'variables': {}, 'dimensions': {}, 'groups': { 'nest1': { 'variables': {}, 'dimensions': {}, 'groups': { 'nest2': { 'variables': {}, 'dimensions': {}, 'groups': { 'nest1': { 'variables': {}, 'dimensions': { 'outlier': Dimension(name='outlier', size=4, size_current=4, dist=False, src_idx='auto') }, 'groups': {} } } } } } } } } self.assertEqual(actual[MPI_RANK], desired) if k.dim_count == 2 and k.nested: self.assertIsNotNone(driver.metadata_source['groups']['nest1'] ['groups']['nest2']) two_dimensions = [ Dimension(name='one', size=1, size_current=1), Dimension(name='two', size=2, size_current=2) ] nest1 = {'dimensions': two_dimensions, 'groups': {}} template = deepcopy(nest1) nest1['groups']['nest2'] = deepcopy(template) nest1['groups']['nest2']['groups']['nest1'] = deepcopy( template) nest1['groups']['nest2']['groups']['nest3'] = deepcopy( template) nest1['groups']['nest2']['groups']['nest1'][ 'dimensions'].append(Dimension('outlier', 4)) desired = { None: { 'dimensions': two_dimensions, 'groups': { 'nest1': nest1 } } } groups_actual = list(iter_all_group_keys((actual[MPI_RANK]))) groups_desired = list(iter_all_group_keys(desired)) self.assertEqual(groups_actual, groups_desired)