def insert_weighted(index_path, dst_wd, dst_master_path): """ Inserted weighted, destination variable data into the master destination file. :param str index_path: Path to the split index netCDF file. :param str dst_wd: Working directory containing the destination files holding the weighted data. :param str dst_master_path: Path to the destination master weight file. """ index_field = RequestDataset(index_path).get() gs_index_v = index_field[ GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE] dst_filenames = gs_index_v.attrs[ GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] dst_filenames = index_field[dst_filenames] y_bounds = GridChunkerConstants.IndexFile.NAME_Y_DST_BOUNDS_VARIABLE y_bounds = gs_index_v.attrs[y_bounds] y_bounds = index_field[y_bounds].get_value() x_bounds = GridChunkerConstants.IndexFile.NAME_X_DST_BOUNDS_VARIABLE x_bounds = gs_index_v.attrs[x_bounds] x_bounds = index_field[x_bounds].get_value() joined = dst_filenames.join_string_value() dst_master_field = RequestDataset(dst_master_path).get() for data_variable in dst_master_field.data_variables: assert data_variable.ndim == 3 assert not data_variable.has_allocated_value for time_index in range(dst_master_field.time.shape[0]): for vidx, source_path in enumerate(joined): source_path = os.path.join(dst_wd, source_path) slc = { dst_master_field.time.dimensions[0].name: time_index, dst_master_field.y.dimensions[0].name: slice(None), dst_master_field.x.dimensions[0].name: slice(None) } source_data = RequestDataset(source_path).get()[ data_variable.name][slc] assert not source_data.has_allocated_value with nc.Dataset(dst_master_path, 'a') as ds: ds.variables[data_variable.name][ time_index, y_bounds[vidx][0]:y_bounds[vidx][1], x_bounds[vidx][0]:x_bounds[vidx] [1]] = source_data.get_value()
def insert_weighted(index_path, dst_wd, dst_master_path, data_variables='auto'): """ Inserted weighted, destination variable data into the master destination file. :param str index_path: Path to the split index netCDF file. :param str dst_wd: Working directory containing the destination files holding the weighted data. :param str dst_master_path: Path to the destination master weight file. :param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used. """ if vm.size > 1: raise NotImplementedError('serial only') index_field = RequestDataset(index_path).get() gs_index_v = index_field[ GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE] dst_filenames = gs_index_v.attrs[ GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] dst_filenames = index_field[dst_filenames] y_bounds = GridChunkerConstants.IndexFile.NAME_Y_DST_BOUNDS_VARIABLE y_bounds = gs_index_v.attrs[y_bounds] y_bounds = index_field[y_bounds].get_value() x_bounds = GridChunkerConstants.IndexFile.NAME_X_DST_BOUNDS_VARIABLE x_bounds = gs_index_v.attrs[x_bounds] x_bounds = index_field[x_bounds].get_value() joined = dst_filenames.join_string_value() if data_variables == 'auto': v = None else: v = data_variables dst_master_field = RequestDataset(dst_master_path, variable=v).get() for data_variable in dst_master_field.data_variables: assert not data_variable.has_allocated_value if data_variable.ndim == 3: for time_index in range(dst_master_field.time.shape[0]): for vidx, source_path in enumerate(joined): source_path = os.path.join(dst_wd, source_path) slc = { dst_master_field.time.dimensions[0].name: time_index, dst_master_field.y.dimensions[0].name: slice(None), dst_master_field.x.dimensions[0].name: slice(None) } source_field = RequestDataset( source_path).create_field() try: source_data = source_field[data_variable.name][slc] except KeyError: if data_variable.name not in source_field.keys(): msg = "The destination variable '{}' is not in the destination file '{}'. Was SMM applied?".format( data_variable.name, source_path) raise KeyError(msg) else: raise assert not source_data.has_allocated_value with nc.Dataset(dst_master_path, 'a') as ds: ds.variables[data_variable.name][ time_index, y_bounds[vidx][0]:y_bounds[vidx][1], x_bounds[vidx][0]:x_bounds[vidx] [1]] = source_data.get_value() elif data_variable.ndim == 2: for vidx, source_path in enumerate(joined): source_path = os.path.join(dst_wd, source_path) source_data = RequestDataset(source_path).get()[ data_variable.name] assert not source_data.has_allocated_value with nc.Dataset(dst_master_path, 'a') as ds: ds.variables[data_variable.name][ y_bounds[vidx][0]:y_bounds[vidx][1], x_bounds[vidx] [0]:x_bounds[vidx][1]] = source_data.get_value() else: raise NotImplementedError(data_variable.ndim)
def insert_weighted(index_path, dst_wd, dst_master_path, data_variables='auto'): """ Inserted weighted, destination variable data into the master destination file. :param str index_path: Path to the split index netCDF file. :param str dst_wd: Working directory containing the destination files holding the weighted data. :param str dst_master_path: Path to the destination master weight file. :param list data_variables: Optional list of data variables. Otherwise, auto-discovery is used. """ if vm.size > 1: raise NotImplementedError('serial only') index_field = RequestDataset(index_path).get() gs_index_v = index_field[GridChunkerConstants.IndexFile.NAME_INDEX_VARIABLE] dst_filenames = gs_index_v.attrs[GridChunkerConstants.IndexFile.NAME_DESTINATION_VARIABLE] dst_filenames = index_field[dst_filenames] y_bounds = GridChunkerConstants.IndexFile.NAME_Y_DST_BOUNDS_VARIABLE y_bounds = gs_index_v.attrs[y_bounds] y_bounds = index_field[y_bounds].get_value() x_bounds = GridChunkerConstants.IndexFile.NAME_X_DST_BOUNDS_VARIABLE x_bounds = gs_index_v.attrs[x_bounds] x_bounds = index_field[x_bounds].get_value() joined = dst_filenames.join_string_value() if data_variables == 'auto': v = None else: v = data_variables dst_master_field = RequestDataset(dst_master_path, variable=v).get() for data_variable in dst_master_field.data_variables: assert not data_variable.has_allocated_value if data_variable.ndim == 3: for time_index in range(dst_master_field.time.shape[0]): for vidx, source_path in enumerate(joined): source_path = os.path.join(dst_wd, source_path) slc = {dst_master_field.time.dimensions[0].name: time_index, dst_master_field.y.dimensions[0].name: slice(None), dst_master_field.x.dimensions[0].name: slice(None)} source_field = RequestDataset(source_path).create_field() try: source_data = source_field[data_variable.name][slc] except KeyError: if data_variable.name not in source_field.keys(): msg = "The destination variable '{}' is not in the destination file '{}'. Was SMM applied?".format( data_variable.name, source_path) raise KeyError(msg) else: raise assert not source_data.has_allocated_value with nc.Dataset(dst_master_path, 'a') as ds: ds.variables[data_variable.name][time_index, y_bounds[vidx][0]:y_bounds[vidx][1], x_bounds[vidx][0]:x_bounds[vidx][1]] = source_data.get_value() elif data_variable.ndim == 2: for vidx, source_path in enumerate(joined): source_path = os.path.join(dst_wd, source_path) source_data = RequestDataset(source_path).get()[data_variable.name] assert not source_data.has_allocated_value with nc.Dataset(dst_master_path, 'a') as ds: ds.variables[data_variable.name][y_bounds[vidx][0]:y_bounds[vidx][1], x_bounds[vidx][0]:x_bounds[vidx][1]] = source_data.get_value() else: raise NotImplementedError(data_variable.ndim)