def chunked_smm(wd, index_path, insert_weighted, destination, data_variables): if wd is None: wd = os.getcwd() if data_variables != 'auto': data_variables = data_variables.split(',') if index_path is None: index_path = os.path.join( wd, constants.GridChunkerConstants.DEFAULT_PATHS['index_file']) ocgis.vm.barrier() assert os.path.exists(index_path) if insert_weighted: if destination is None: raise ValueError( 'If --insert_weighted, then "destination" must be provided.') # ------------------------------------------------------------------------------------------------------------------ GridChunker.smm(index_path, wd, data_variables=data_variables) if insert_weighted: with ocgis.vm.scoped_barrier(first=True, last=True): with ocgis.vm.scoped('insert weighted', [0]): if not ocgis.vm.is_null: GridChunker.insert_weighted(index_path, wd, destination, data_variables=data_variables)
def chunked_smm(wd, index_path, insert_weighted, destination, data_variables): if wd is None: wd = os.getcwd() if data_variables != 'auto': data_variables = data_variables.split(',') if index_path is None: index_path = os.path.join(wd, constants.GridChunkerConstants.DEFAULT_PATHS['index_file']) ocgis.vm.barrier() assert os.path.exists(index_path) if insert_weighted: if destination is None: raise ValueError('If --insert_weighted, then "destination" must be provided.') # ------------------------------------------------------------------------------------------------------------------ GridChunker.smm(index_path, wd, data_variables=data_variables) if insert_weighted: with ocgis.vm.scoped_barrier(first=True, last=True): with ocgis.vm.scoped('insert weighted', [0]): if not ocgis.vm.is_null: GridChunker.insert_weighted(index_path, wd, destination, data_variables=data_variables)
def test_write_esmf_weights(self): # Create source and destination fields. This is the identity test, so the source and destination fields are # equivalent. src_grid = create_gridxy_global(resolution=3.0, crs=Spherical()) # Only test masking in serial to make indexing easier...just being lazy if vm.size == 1: mask = src_grid.get_mask(create=True) mask[4, 5] = True mask[25, 27] = True src_grid.set_mask(mask) self.assertEqual(src_grid.get_mask().sum(), 2) src_field = create_exact_field(src_grid, 'foo', ntime=3) dst_field = deepcopy(src_field) # Write the fields to disk for use in global file reconstruction and testing. if vm.rank == 0: master_path = self.get_temporary_file_path('foo.nc') src_field_path = self.get_temporary_file_path('src_field.nc') else: master_path = None src_field_path = None master_path = vm.bcast(master_path) src_field_path = vm.bcast(src_field_path) assert not os.path.exists(master_path) dst_field.write(master_path) src_field.write(src_field_path) # Remove the destination data variable to test its creation and filling dst_field.remove_variable('foo') # Chunk the fields and generate weights paths = {'wd': self.current_dir_output} gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths, esmf_kwargs={'regrid_method': 'BILINEAR'}) gc.write_chunks() # This is the path to the index file describing how to reconstruct the grid file index_path = os.path.join(self.current_dir_output, gc.paths['index_file']) # Execute the sparse matrix multiplication using weights read from file gc.smm(index_path, paths['wd']) with vm.scoped('index and reconstruct', [0]): if not vm.is_null: # Reconstruct the global destination file gc.insert_weighted(index_path, self.current_dir_output, master_path) # Load the actual values from file (destination) actual_field = RequestDataset(master_path).create_field() actual = actual_field.data_variables[0].mv() # Load the desired data from file (original values in the source field) desired = RequestDataset(src_field_path).create_field().data_variables[0].mv() if vm.size_global == 1: # Masking is only tested in serial self.assertEqual(actual_field.grid.get_mask().sum(), 2) else: self.assertIsNone(actual_field.grid.get_mask()) self.assertNumpyAll(actual, desired)
def test_write_esmf_weights(self): # Create source and destination fields. This is the identity test, so the source and destination fields are # equivalent. src_grid = create_gridxy_global(resolution=3.0, crs=Spherical()) # Only test masking in serial to make indexing easier...just being lazy if vm.size == 1: mask = src_grid.get_mask(create=True) mask[4, 5] = True mask[25, 27] = True src_grid.set_mask(mask) self.assertEqual(src_grid.get_mask().sum(), 2) src_field = create_exact_field(src_grid, 'foo', ntime=3) dst_field = deepcopy(src_field) # Write the fields to disk for use in global file reconstruction and testing. if vm.rank == 0: master_path = self.get_temporary_file_path('foo.nc') src_field_path = self.get_temporary_file_path('src_field.nc') else: master_path = None src_field_path = None master_path = vm.bcast(master_path) src_field_path = vm.bcast(src_field_path) assert not os.path.exists(master_path) dst_field.write(master_path) src_field.write(src_field_path) # Remove the destination data variable to test its creation and filling dst_field.remove_variable('foo') # Chunk the fields and generate weights paths = {'wd': self.current_dir_output} gc = GridChunker(src_field, dst_field, nchunks_dst=(2, 2), genweights=True, paths=paths, esmf_kwargs={'regrid_method': 'BILINEAR'}) gc.write_chunks() # This is the path to the index file describing how to reconstruct the grid file index_path = os.path.join(self.current_dir_output, gc.paths['index_file']) # Execute the sparse matrix multiplication using weights read from file gc.smm(index_path, paths['wd']) with vm.scoped('index and reconstruct', [0]): if not vm.is_null: # Reconstruct the global destination file gc.insert_weighted(index_path, self.current_dir_output, master_path) # Load the actual values from file (destination) actual_field = RequestDataset(master_path).create_field() actual = actual_field.data_variables[0].mv() # Load the desired data from file (original values in the source field) desired = RequestDataset( src_field_path).create_field().data_variables[0].mv() if vm.size_global == 1: # Masking is only tested in serial self.assertEqual(actual_field.grid.get_mask().sum(), 2) else: self.assertIsNone(actual_field.grid.get_mask()) self.assertNumpyAll(actual, desired)