def test_convert_neg(self): a = np.arange(16).reshape((4, 4)) a[2, 3] = -5 a[0, 1] = -7 b = convert_neg(a.copy(), a.size) self.assertEqual(b[2, 3], 11) self.assertEqual(b[0, 1], 9) inds = np.where(a != b) self.assertTrue(np.all(inds[0] == np.array([0, 2]))) self.assertTrue(np.all(inds[1] == np.array([1, 3])))
def _setup_transfers(group, recurse=True): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. recurse : bool Whether to call this method in subsystems. """ group._transfers = {} iproc = group.comm.rank rev = group._mode == 'rev' or group._mode == 'auto' def merge(indices_list): if len(indices_list) > 0: return np.concatenate(indices_list) else: return _empty_idx_array if recurse: for subsys in group._subgroups_myproc: subsys._setup_transfers(recurse) # Pre-compute map from abs_names to the index of the containing subsystem abs2isub = {} for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds): for type_ in ['input', 'output']: for abs_name in subsys._var_allprocs_abs_names[type_]: abs2isub[abs_name] = isub abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta transfers = group._transfers vectors = group._vectors offsets = _global2local_offsets(group._get_var_offsets()) for vec_name in group._lin_rel_vec_name_list: relvars, _ = group._relevant[vec_name]['@all'] relvars_in = relvars['input'] relvars_out = relvars['output'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = [] xfer_out = [] fwd_xfer_in = [[] for i in range(nsub_allprocs)] fwd_xfer_out = [[] for i in range(nsub_allprocs)] if rev: rev_xfer_in = [[] for i in range(nsub_allprocs)] rev_xfer_out = [[] for i in range(nsub_allprocs)] allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all explicit / implicit connections owned by this system for abs_in, abs_out in iteritems(group._conn_abs_in2out): if abs_out not in relvars_out or abs_in not in relvars_in: continue # Only continue if the input exists on this processor if abs_in in abs2meta and abs_in in relvars['input']: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices shape_in = meta_in['shape'] shape_out = meta_out['shape'] global_size_out = meta_out['global_size'] src_indices = meta_in['src_indices'] if src_indices is None: pass elif src_indices.ndim == 1: src_indices = convert_neg(src_indices, global_size_out) else: if len(shape_out ) == 1 or shape_in == src_indices.shape: src_indices = src_indices.flatten() src_indices = convert_neg(src_indices, global_size_out) else: # TODO: this duplicates code found # in System._setup_scaling. entries = [list(range(x)) for x in shape_in] cols = np.vstack(src_indices[i] for i in product(*entries)) dimidxs = [ convert_neg(cols[:, i], shape_out[i]) for i in range(cols.shape[1]) ] src_indices = np.ravel_multi_index( dimidxs, shape_out) # 1. Compute the output indices offset = offsets_out[iproc, idx_out] if src_indices is None: output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = src_indices + offset # 2. Compute the input indices input_inds = np.arange(offsets_in[iproc, idx_in], offsets_in[iproc, idx_in] + sizes_in[iproc, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds xfer_in.append(input_inds) xfer_out.append(output_inds) isub = abs2isub[abs_in] fwd_xfer_in[isub].append(input_inds) fwd_xfer_out[isub].append(output_inds) if rev and abs_out in abs2isub: isub = abs2isub[abs_out] rev_xfer_in[isub].append(input_inds) rev_xfer_out[isub].append(output_inds) xfer_in = merge(xfer_in) xfer_out = merge(xfer_out) for isub in range(nsub_allprocs): fwd_xfer_in[isub] = merge(fwd_xfer_in[isub]) fwd_xfer_out[isub] = merge(fwd_xfer_out[isub]) if rev: rev_xfer_in[isub] = merge(rev_xfer_in[isub]) rev_xfer_out[isub] = merge(rev_xfer_out[isub]) out_vec = vectors['output'][vec_name] transfers[vec_name] = {} xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): transfers[vec_name]['fwd', isub] = DefaultTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) if rev: transfers[vec_name]['rev', isub] = DefaultTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) transfers['nonlinear'] = transfers['linear']
def _setup_driver(self, problem): """ Prepare the driver for execution. This is the final thing to run during setup. Parameters ---------- problem : <Problem> Pointer to the containing problem. """ self._problem = weakref.ref(problem) model = problem.model self._total_jac = None self._has_scaling = (np.any( [r['total_scaler'] is not None for r in self._responses.values()]) or np.any([ dv['total_scaler'] is not None for dv in self._designvars.values() ])) # Determine if any design variables are discrete. self._designvars_discrete = [ name for name, meta in self._designvars.items() if meta['ivc_source'] in model._discrete_outputs ] if not self.supports['integer_design_vars'] and len( self._designvars_discrete) > 0: msg = "Discrete design variables are not supported by this driver: " msg += '.'.join(self._designvars_discrete) raise RuntimeError(msg) con_set = set() obj_set = set() dv_set = set() self._remote_dvs = remote_dv_dict = {} self._remote_cons = remote_con_dict = {} self._dist_driver_vars = dist_dict = {} self._remote_objs = remote_obj_dict = {} src_design_vars = prom2ivc_src_dict(self._designvars) src_cons = prom2ivc_src_dict(self._cons) src_objs = prom2ivc_src_dict(self._objs) responses = prom2ivc_src_dict(self._responses) # Now determine if later we'll need to allgather cons, objs, or desvars. if model.comm.size > 1 and model._subsystems_allprocs: local_out_vars = set(model._outputs._abs_iter()) remote_dvs = set(src_design_vars) - local_out_vars remote_cons = set(src_cons) - local_out_vars remote_objs = set(src_objs) - local_out_vars all_remote_vois = model.comm.allgather( (remote_dvs, remote_cons, remote_objs)) for rem_dvs, rem_cons, rem_objs in all_remote_vois: con_set.update(rem_cons) obj_set.update(rem_objs) dv_set.update(rem_dvs) # If we have remote VOIs, pick an owning rank for each and use that # to bcast to others later owning_ranks = model._owning_rank sizes = model._var_sizes['nonlinear']['output'] abs2meta = model._var_allprocs_abs2meta rank = model.comm.rank nprocs = model.comm.size for i, vname in enumerate(model._var_allprocs_abs_names['output']): if vname in responses: indices = responses[vname].get('indices') elif vname in src_design_vars: indices = src_design_vars[vname].get('indices') else: continue if abs2meta[vname]['distributed']: idx = model._var_allprocs_abs2idx['nonlinear'][vname] dist_sizes = model._var_sizes['nonlinear']['output'][:, idx] total_dist_size = np.sum(dist_sizes) # Determine which indices are on our proc. offsets = sizes2offsets(dist_sizes) if indices is not None: indices = convert_neg(indices, total_dist_size) true_sizes = np.zeros(nprocs, dtype=INT_DTYPE) for irank in range(nprocs): dist_inds = indices[np.logical_and( indices >= offsets[irank], indices < (offsets[irank] + dist_sizes[irank]))] if irank == rank: local_indices = dist_inds - offsets[rank] distrib_indices = dist_inds true_sizes[irank] = dist_inds.size dist_dict[vname] = (local_indices, true_sizes, distrib_indices) else: dist_dict[vname] = (_full_slice, dist_sizes, slice( offsets[rank], offsets[rank] + dist_sizes[rank])) else: owner = owning_ranks[vname] sz = sizes[owner, i] if vname in dv_set: remote_dv_dict[vname] = (owner, sz) if vname in con_set: remote_con_dict[vname] = (owner, sz) if vname in obj_set: remote_obj_dict[vname] = (owner, sz) self._remote_responses = self._remote_cons.copy() self._remote_responses.update(self._remote_objs) # set up simultaneous deriv coloring if coloring_mod._use_total_sparsity: # reset the coloring if self._coloring_info['dynamic'] or self._coloring_info[ 'static'] is not None: self._coloring_info['coloring'] = None coloring = self._get_static_coloring() if coloring is not None and self.supports[ 'simultaneous_derivatives']: if model._owns_approx_jac: coloring._check_config_partial(model) else: coloring._check_config_total(self) self._setup_simul_coloring()
def _setup_transfers(group): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. """ rev = group._mode != 'fwd' for subsys in group._subgroups_myproc: subsys._setup_transfers() abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta myproc = group.comm.rank transfers = group._transfers = {} vectors = group._vectors offsets = group._get_var_offsets() vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names mypathlen = len(group.pathname + '.' if group.pathname else '') sub_inds = group._subsystems_inds for vec_name in vec_names: relvars, _ = group._relevant[vec_name]['@all'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = [] xfer_out = [] fwd_xfer_in = [[] for s in group._subsystems_allprocs] fwd_xfer_out = [[] for s in group._subsystems_allprocs] if rev: rev_xfer_in = [[] for s in group._subsystems_allprocs] rev_xfer_out = [[] for s in group._subsystems_allprocs] allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all connections owned by this system for abs_in, abs_out in group._conn_abs_in2out.items(): if abs_out not in relvars['output'] or abs_in not in relvars[ 'input']: continue # Only continue if the input exists on this processor if abs_in in abs2meta: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: owner = group._owning_rank[abs_out] # if the input is larger than the output on a single proc, we have # to just loop over the procs in the same way we do when src_indices # is defined. if meta_in['size'] > sizes_out[owner, idx_out]: src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) elif src_indices.ndim == 1: if _is_slice(src_indices): indices = _slice_indices(src_indices, meta_out['global_size'], meta_out['global_shape']) src_indices = convert_neg(indices, meta_out['global_size']) else: src_indices = _flatten_src_indices( src_indices, meta_in['shape'], meta_out['global_shape'], meta_out['global_size']) # 1. Compute the output indices # NOTE: src_indices are relative to a single, possibly distributed variable, # while the output_inds that we compute are relative to the full distributed # array that contains all local variables from each rank stacked in rank order. if src_indices is None: if meta_out['distributed']: # input in this case is non-distributed (else src_indices would be # defined by now). The input size must match the full # distributed size of the output. for rank, sz in enumerate(sizes_out[:, idx_out]): if sz > 0: out_offset = offsets_out[rank, idx_out] break output_inds = np.arange(out_offset, out_offset + meta_out['global_size'], dtype=INT_DTYPE) else: rank = myproc if abs_out in abs2meta else owner offset = offsets_out[rank, idx_out] output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = np.zeros(src_indices.size, INT_DTYPE) start = end = 0 for iproc in range(group.comm.size): end += sizes_out[iproc, idx_out] if start == end: continue # The part of src on iproc on_iproc = np.logical_and(start <= src_indices, src_indices < end) if np.any(on_iproc): # This converts from iproc-then-ivar to ivar-then-iproc ordering # Subtract off part of previous procs # Then add all variables on previous procs # Then all previous variables on this proc # - np.sum(out_sizes[:iproc, idx_out]) # + np.sum(out_sizes[:iproc, :]) # + np.sum(out_sizes[iproc, :idx_out]) # + inds offset = offsets_out[iproc, idx_out] - start output_inds[ on_iproc] = src_indices[on_iproc] + offset start = end # 2. Compute the input indices input_inds = np.arange(offsets_in[myproc, idx_in], offsets_in[myproc, idx_in] + sizes_in[myproc, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].split('.', 1)[0] isub = sub_inds[sub_in] fwd_xfer_in[isub].append(input_inds) fwd_xfer_out[isub].append(output_inds) if rev: sub_out = abs_out[mypathlen:].split('.', 1)[0] isub = sub_inds[sub_out] rev_xfer_in[isub].append(input_inds) rev_xfer_out[isub].append(output_inds) transfers[vec_name] = {} for isub in range(nsub_allprocs): fwd_xfer_in[isub] = _merge(fwd_xfer_in[isub]) fwd_xfer_out[isub] = _merge(fwd_xfer_out[isub]) if rev: rev_xfer_in[isub] = _merge(rev_xfer_in[isub]) rev_xfer_out[isub] = _merge(rev_xfer_out[isub]) xfer_in = np.concatenate(fwd_xfer_in) xfer_out = np.concatenate(fwd_xfer_out) out_vec = vectors['output'][vec_name] xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): transfers[vec_name]['fwd', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) if rev: transfers[vec_name]['rev', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) if group._use_derivatives: transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. recurse : bool Whether to call this method in subsystems. """ group._transfers = {} iproc = group.comm.rank rev = group._mode == 'rev' or group._mode == 'auto' def merge(indices_list): if len(indices_list) > 0: return np.concatenate(indices_list) else: return _empty_idx_array if recurse: for subsys in group._subgroups_myproc: subsys._setup_transfers(recurse) # Pre-compute map from abs_names to the index of the containing subsystem abs2isub = {} for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds): for type_ in ['input', 'output']: for abs_name in subsys._var_allprocs_abs_names[type_]: abs2isub[abs_name] = isub abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta transfers = group._transfers vectors = group._vectors offsets = _global2local_offsets(group._get_var_offsets()) vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names for vec_name in vec_names: relvars, _ = group._relevant[vec_name]['@all'] relvars_in = relvars['input'] relvars_out = relvars['output'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = [] xfer_out = [] fwd_xfer_in = [[] for i in range(nsub_allprocs)] fwd_xfer_out = [[] for i in range(nsub_allprocs)] if rev: rev_xfer_in = [[] for i in range(nsub_allprocs)] rev_xfer_out = [[] for i in range(nsub_allprocs)] allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all explicit / implicit connections owned by this system for abs_in, abs_out in iteritems(group._conn_abs_in2out): if abs_out not in relvars_out or abs_in not in relvars_in: continue # Only continue if the input exists on this processor if abs_in in abs2meta and abs_in in relvars['input']: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: pass elif src_indices.ndim == 1: src_indices = convert_neg(src_indices, meta_out['global_size']) else: src_indices = _flatten_src_indices(src_indices, meta_in['shape'], meta_out['global_shape'], meta_out['global_size']) # 1. Compute the output indices offset = offsets_out[iproc, idx_out] if src_indices is None: output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = src_indices + offset # 2. Compute the input indices input_inds = np.arange(offsets_in[iproc, idx_in], offsets_in[iproc, idx_in] + sizes_in[iproc, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds xfer_in.append(input_inds) xfer_out.append(output_inds) isub = abs2isub[abs_in] fwd_xfer_in[isub].append(input_inds) fwd_xfer_out[isub].append(output_inds) if rev and abs_out in abs2isub: isub = abs2isub[abs_out] rev_xfer_in[isub].append(input_inds) rev_xfer_out[isub].append(output_inds) xfer_in = merge(xfer_in) xfer_out = merge(xfer_out) for isub in range(nsub_allprocs): fwd_xfer_in[isub] = merge(fwd_xfer_in[isub]) fwd_xfer_out[isub] = merge(fwd_xfer_out[isub]) if rev: rev_xfer_in[isub] = merge(rev_xfer_in[isub]) rev_xfer_out[isub] = merge(rev_xfer_out[isub]) out_vec = vectors['output'][vec_name] transfers[vec_name] = {} xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): transfers[vec_name]['fwd', isub] = DefaultTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) if rev: transfers[vec_name]['rev', isub] = DefaultTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) if group._use_derivatives: transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. recurse : bool Whether to call this method in subsystems. """ rev = group._mode == 'rev' or group._mode == 'auto' def merge(indices_list): if len(indices_list) > 0: return np.concatenate(indices_list) else: return _empty_idx_array if recurse: for subsys in group._subgroups_myproc: subsys._setup_transfers(recurse) # Pre-compute map from abs_names to the index of the containing subsystem abs2isub = {} for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds): for type_ in ['input', 'output']: for abs_name in subsys._var_allprocs_abs_names[type_]: abs2isub[abs_name] = isub abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta myproc = group.comm.rank transfers = group._transfers = {} vectors = group._vectors offsets = group._get_var_offsets() vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names for vec_name in vec_names: relvars, _ = group._relevant[vec_name]['@all'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = [] xfer_out = [] fwd_xfer_in = [[] for i in range(nsub_allprocs)] fwd_xfer_out = [[] for i in range(nsub_allprocs)] if rev: rev_xfer_in = [[] for i in range(nsub_allprocs)] rev_xfer_out = [[] for i in range(nsub_allprocs)] allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all connections owned by this system for abs_in, abs_out in iteritems(group._conn_abs_in2out): if abs_out not in relvars['output']: continue # Only continue if the input exists on this processor if abs_in in abs2meta and abs_in in relvars['input']: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: owner = group._owning_rank[abs_out] # if the input is larger than the output on a single proc, we have # to just loop over the procs in the same way we do when src_indices # is defined. if meta_in['size'] > sizes_out[owner, idx_out]: src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) elif src_indices.ndim == 1: src_indices = convert_neg(src_indices, meta_out['global_size']) else: src_indices = _flatten_src_indices( src_indices, meta_in['shape'], meta_out['global_shape'], meta_out['global_size']) # 1. Compute the output indices if src_indices is None: start = 0 if owner == 0 else np.sum(sizes_out[:owner, idx_out]) offset = offsets_out[owner, idx_out] - start output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = np.zeros(src_indices.size, INT_DTYPE) start = end = 0 for iproc in range(group.comm.size): end += sizes_out[iproc, idx_out] # The part of src on iproc on_iproc = np.logical_and(start <= src_indices, src_indices < end) if np.any(on_iproc): # This converts from iproc-then-ivar to ivar-then-iproc ordering # Subtract off part of previous procs # Then add all variables on previous procs # Then all previous variables on this proc # - np.sum(out_sizes[:iproc, idx_out]) # + np.sum(out_sizes[:iproc, :]) # + np.sum(out_sizes[iproc, :idx_out]) # + inds offset = offsets_out[iproc, idx_out] - start output_inds[ on_iproc] = src_indices[on_iproc] + offset start = end # 2. Compute the input indices input_inds = np.arange(offsets_in[myproc, idx_in], offsets_in[myproc, idx_in] + sizes_in[myproc, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds xfer_in.append(input_inds) xfer_out.append(output_inds) isub = abs2isub[abs_in] fwd_xfer_in[isub].append(input_inds) fwd_xfer_out[isub].append(output_inds) if rev and abs_out in abs2isub: isub = abs2isub[abs_out] rev_xfer_in[isub].append(input_inds) rev_xfer_out[isub].append(output_inds) xfer_in = merge(xfer_in) xfer_out = merge(xfer_out) for isub in range(nsub_allprocs): fwd_xfer_in[isub] = merge(fwd_xfer_in[isub]) fwd_xfer_out[isub] = merge(fwd_xfer_out[isub]) if rev: rev_xfer_in[isub] = merge(rev_xfer_in[isub]) rev_xfer_out[isub] = merge(rev_xfer_out[isub]) out_vec = vectors['output'][vec_name] transfers[vec_name] = {} xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): transfers[vec_name]['fwd', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) if rev: transfers[vec_name]['rev', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) if group._use_derivatives: transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. recurse : bool Whether to call this method in subsystems. """ iproc = group.comm.rank rev = group._mode == 'rev' or group._mode == 'auto' if recurse: for subsys in group._subgroups_myproc: subsys._setup_transfers(recurse) abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta group._transfers = transfers = {} vectors = group._vectors offsets = _global2local_offsets(group._get_var_offsets()) vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names mypathlen = len(group.pathname + '.' if group.pathname else '') sub_inds = group._subsystems_inds for vec_name in vec_names: relvars, _ = group._relevant[vec_name]['@all'] relvars_in = relvars['input'] relvars_out = relvars['output'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = [] xfer_out = [] fwd_xfer_in = [[] for s in group._subsystems_allprocs] fwd_xfer_out = [[] for s in group._subsystems_allprocs] if rev: rev_xfer_in = [[] for s in group._subsystems_allprocs] rev_xfer_out = [[] for s in group._subsystems_allprocs] allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all connections owned by this group for abs_in, abs_out in group._conn_abs_in2out.items(): if abs_out not in relvars_out or abs_in not in relvars_in: continue # Only continue if the input exists on this processor if abs_in in abs2meta: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: pass elif src_indices.ndim == 1: src_indices = convert_neg(src_indices, meta_out['global_size']) else: src_indices = _flatten_src_indices( src_indices, meta_in['shape'], meta_out['global_shape'], meta_out['global_size']) # 1. Compute the output indices offset = offsets_out[iproc, idx_out] if src_indices is None: output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = src_indices + offset # 2. Compute the input indices input_inds = np.arange(offsets_in[iproc, idx_in], offsets_in[iproc, idx_in] + sizes_in[iproc, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].split('.', 1)[0] isub = sub_inds[sub_in] fwd_xfer_in[isub].append(input_inds) fwd_xfer_out[isub].append(output_inds) if rev and abs_out in abs2meta: sub_out = abs_out[mypathlen:].split('.', 1)[0] isub = sub_inds[sub_out] rev_xfer_in[isub].append(input_inds) rev_xfer_out[isub].append(output_inds) tot_size = 0 for isub in range(nsub_allprocs): fwd_xfer_in[isub] = _merge(fwd_xfer_in[isub]) fwd_xfer_out[isub] = _merge(fwd_xfer_out[isub]) tot_size += fwd_xfer_in[isub].size if rev: rev_xfer_in[isub] = _merge(rev_xfer_in[isub]) rev_xfer_out[isub] = _merge(rev_xfer_out[isub]) transfers[vec_name] = {} if tot_size > 0: xfer_in = np.concatenate(fwd_xfer_in) xfer_out = np.concatenate(fwd_xfer_out) out_vec = vectors['output'][vec_name] xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) else: xfer_all = None transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): if fwd_xfer_in[isub].size > 0: transfers[vec_name]['fwd', isub] = DefaultTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) else: transfers[vec_name]['fwd', isub] = None if rev: if rev_xfer_out[isub].size > 0: transfers[vec_name]['rev', isub] = DefaultTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) else: transfers[vec_name]['rev', isub] = None if group._use_derivatives: transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. """ iproc = group.comm.rank rev = group._mode == 'rev' or group._mode == 'auto' for subsys in group._subgroups_myproc: subsys._setup_transfers() abs2meta = group._var_abs2meta allprocs_abs2meta_out = group._var_allprocs_abs2meta['output'] group._transfers = transfers = {} vectors = group._vectors offsets = _global2local_offsets(group._get_var_offsets()) vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names allsubs = group._subsystems_allprocs mypathlen = len(group.pathname + '.' if group.pathname else '') for vec_name in vec_names: relvars, _ = group._relevant[vec_name]['@all'] relvars_in = relvars['input'] relvars_out = relvars['output'] # Initialize empty lists for the transfer indices nsub_allprocs = len(allsubs) xfer_in = [] xfer_out = [] fwd_xfer_in = defaultdict(list) fwd_xfer_out = defaultdict(list) if rev: rev_xfer_in = defaultdict(list) rev_xfer_out = defaultdict(list) allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all connections owned by this group for abs_in, abs_out in group._conn_abs_in2out.items(): if abs_out not in relvars_out or abs_in not in relvars_in: continue # Only continue if the input exists on this processor if abs_in in abs2meta['input']: indices = None # Get meta meta_in = abs2meta['input'][abs_in] meta_out = allprocs_abs2meta_out[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is not None: if src_indices.ndim == 1: src_indices = convert_neg(src_indices, meta_out['global_size']) # 1. Compute the output indices offset = offsets_out[iproc, idx_out] if src_indices is None: output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = src_indices + offset # 2. Compute the input indices input_inds = np.arange(offsets_in[iproc, idx_in], offsets_in[iproc, idx_in] + sizes_in[iproc, idx_in], dtype=INT_DTYPE) if indices is not None: input_inds = input_inds.reshape(indices.shape) # Now the indices are ready - input_inds, output_inds sub_in = abs_in[mypathlen:].split('.', 1)[0] fwd_xfer_in[sub_in].append(input_inds) fwd_xfer_out[sub_in].append(output_inds) if rev and abs_out in abs2meta['output']: sub_out = abs_out[mypathlen:].split('.', 1)[0] rev_xfer_in[sub_out].append(input_inds) rev_xfer_out[sub_out].append(output_inds) tot_size = 0 for sname, inds in fwd_xfer_in.items(): fwd_xfer_in[sname] = arr = _merge(inds) fwd_xfer_out[sname] = _merge(fwd_xfer_out[sname]) tot_size += arr.size if rev: for sname, inds in rev_xfer_in.items(): rev_xfer_in[sname] = _merge(inds) rev_xfer_out[sname] = _merge(rev_xfer_out[sname]) transfers[vec_name] = {} if tot_size > 0: try: xfer_in = np.concatenate(list(fwd_xfer_in.values())) xfer_out = np.concatenate(list(fwd_xfer_out.values())) except ValueError: xfer_in = xfer_out = np.zeros(0, dtype=INT_DTYPE) out_vec = vectors['output'][vec_name] xfer_all = DefaultTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) else: xfer_all = None transfers[vec_name]['fwd'] = xfwd = {} xfwd[None] = xfer_all if rev: transfers[vec_name]['rev'] = xrev = {} xrev[None] = xfer_all for sname, inds in fwd_xfer_in.items(): if inds.size > 0: xfwd[sname] = DefaultTransfer(vectors['input'][vec_name], vectors['output'][vec_name], inds, fwd_xfer_out[sname], group.comm) else: xfwd[sname] = None if rev: for sname, inds in rev_xfer_out.items(): if inds.size > 0: xrev[sname] = DefaultTransfer(vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[sname], inds, group.comm) else: xrev[sname] = None if group._use_derivatives: transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. recurse : bool Whether to call this method in subsystems. """ group._transfers = {} rev = group._mode == 'rev' def merge(indices_list): if len(indices_list) > 0: return np.concatenate(indices_list) else: return _empty_idx_array if recurse: for subsys in group._subgroups_myproc: subsys._setup_transfers(recurse) # Pre-compute map from abs_names to the index of the containing subsystem abs2isub = {} for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds): for type_ in ['input', 'output']: for abs_name in subsys._var_allprocs_abs_names[type_]: abs2isub[abs_name] = isub abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta myproc = group.comm.rank transfers = group._transfers vectors = group._vectors for vec_name in group._lin_rel_vec_name_list: relvars, _ = group._relevant[vec_name]['@all'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = {} xfer_out = {} fwd_xfer_in = [{} for i in range(nsub_allprocs)] fwd_xfer_out = [{} for i in range(nsub_allprocs)] if rev: rev_xfer_in = [{} for i in range(nsub_allprocs)] rev_xfer_out = [{} for i in range(nsub_allprocs)] for set_name_in in group._num_var_byset[vec_name]['input']: for set_name_out in group._num_var_byset[vec_name]['output']: key = (set_name_in, set_name_out) xfer_in[key] = [] xfer_out[key] = [] for isub in range(nsub_allprocs): fwd_xfer_in[isub][key] = [] fwd_xfer_out[isub][key] = [] if rev: rev_xfer_in[isub][key] = [] rev_xfer_out[isub][key] = [] allprocs_abs2idx_byset = group._var_allprocs_abs2idx_byset[ vec_name] sizes_byset_in = group._var_sizes_byset[vec_name]['input'] sizes_byset_out = group._var_sizes_byset[vec_name]['output'] offsets = group._get_varset_offsets() offsets_byset_in = offsets[vec_name]['input'] offsets_byset_out = offsets[vec_name]['output'] # Loop through all explicit / implicit connections owned by this system for abs_in, abs_out in iteritems(group._conn_abs_in2out): if abs_out not in relvars['output']: continue # Only continue if the input exists on this processor if abs_in in abs2meta and abs_in in relvars['input']: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] # Get varset info set_name_in = meta_in['var_set'] set_name_out = meta_out['var_set'] idx_byset_in = allprocs_abs2idx_byset[abs_in] idx_byset_out = allprocs_abs2idx_byset[abs_out] # Get the sizes (byset) array sizes_in = sizes_byset_in[set_name_in] sizes_out = sizes_byset_out[set_name_out] offsets_in = offsets_byset_in[set_name_in] offsets_out = offsets_byset_out[set_name_out] # Read in and process src_indices shape_in = meta_in['shape'] shape_out = meta_out['shape'] global_shape_out = meta_out['global_shape'] global_size_out = meta_out['global_size'] src_indices = meta_in['src_indices'] if src_indices is None: owner = group._owning_rank[abs_out] # if the input is larger than the output on a single proc, we have # to just loop over the procs in the same way we do when src_indices # is defined. if meta_in['size'] > sizes_out[owner, idx_byset_out]: src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) elif src_indices.ndim == 1: src_indices = convert_neg(src_indices, global_size_out) else: if len(shape_out ) == 1 or shape_in == src_indices.shape: src_indices = src_indices.flatten() src_indices = convert_neg(src_indices, global_size_out) else: # TODO: this duplicates code found # in System._setup_scaling. entries = [list(range(x)) for x in shape_in] cols = np.vstack(src_indices[i] for i in product(*entries)) dimidxs = [ convert_neg(cols[:, i], global_shape_out[i]) for i in range(cols.shape[1]) ] src_indices = np.ravel_multi_index( dimidxs, global_shape_out) # 1. Compute the output indices if src_indices is None: start = 0 if owner == 0 else np.sum( sizes_out[:owner, idx_byset_out]) offset = offsets_out[owner, idx_byset_out] - start output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = np.zeros(src_indices.size, INT_DTYPE) start = end = 0 for iproc in range(group.comm.size): end += sizes_out[iproc, idx_byset_out] # The part of src on iproc on_iproc = np.logical_and(start <= src_indices, src_indices < end) if np.any(on_iproc): # This converts from iproc-then-ivar to ivar-then-iproc ordering # Subtract off part of previous procs # Then add all variables on previous procs # Then all previous variables on this proc # - np.sum(out_sizes[:iproc, idx_byset_out]) # + np.sum(out_sizes[:iproc, :]) # + np.sum(out_sizes[iproc, :idx_byset_out]) # + inds offset = offsets_out[iproc, idx_byset_out] - start output_inds[ on_iproc] = src_indices[on_iproc] + offset start = end # 2. Compute the input indices input_inds = np.arange(offsets_in[myproc, idx_byset_in], offsets_in[myproc, idx_byset_in] + sizes_in[myproc, idx_byset_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds key = (set_name_in, set_name_out) xfer_in[key].append(input_inds) xfer_out[key].append(output_inds) isub = abs2isub[abs_in] fwd_xfer_in[isub][key].append(input_inds) fwd_xfer_out[isub][key].append(output_inds) if rev and abs_out in abs2isub: isub = abs2isub[abs_out] rev_xfer_in[isub][key].append(input_inds) rev_xfer_out[isub][key].append(output_inds) for set_name_in in group._num_var_byset[vec_name]['input']: for set_name_out in group._num_var_byset[vec_name]['output']: key = (set_name_in, set_name_out) xfer_in[key] = merge(xfer_in[key]) xfer_out[key] = merge(xfer_out[key]) for isub in range(nsub_allprocs): fwd_xfer_in[isub][key] = merge(fwd_xfer_in[isub][key]) fwd_xfer_out[isub][key] = merge( fwd_xfer_out[isub][key]) if rev: rev_xfer_in[isub][key] = merge( rev_xfer_in[isub][key]) rev_xfer_out[isub][key] = merge( rev_xfer_out[isub][key]) out_vec = vectors['output'][vec_name] transfers[vec_name] = {} xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): transfers[vec_name]['fwd', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) if rev: transfers[vec_name]['rev', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) transfers['nonlinear'] = transfers['linear']
def _setup_transfers(group, recurse=True): """ Compute all transfers that are owned by our parent group. Parameters ---------- group : <Group> Parent group. recurse : bool Whether to call this method in subsystems. """ group._transfers = {} rev = group._mode == 'rev' or group._mode == 'auto' def merge(indices_list): if len(indices_list) > 0: return np.concatenate(indices_list) else: return _empty_idx_array if recurse: for subsys in group._subgroups_myproc: subsys._setup_transfers(recurse) # Pre-compute map from abs_names to the index of the containing subsystem abs2isub = {} for subsys, isub in zip(group._subsystems_myproc, group._subsystems_myproc_inds): for type_ in ['input', 'output']: for abs_name in subsys._var_allprocs_abs_names[type_]: abs2isub[abs_name] = isub abs2meta = group._var_abs2meta allprocs_abs2meta = group._var_allprocs_abs2meta myproc = group.comm.rank transfers = group._transfers vectors = group._vectors offsets = group._get_var_offsets() vec_names = group._lin_rel_vec_name_list if group._use_derivatives else group._vec_names for vec_name in vec_names: relvars, _ = group._relevant[vec_name]['@all'] # Initialize empty lists for the transfer indices nsub_allprocs = len(group._subsystems_allprocs) xfer_in = [] xfer_out = [] fwd_xfer_in = [[] for i in range(nsub_allprocs)] fwd_xfer_out = [[] for i in range(nsub_allprocs)] if rev: rev_xfer_in = [[] for i in range(nsub_allprocs)] rev_xfer_out = [[] for i in range(nsub_allprocs)] allprocs_abs2idx = group._var_allprocs_abs2idx[vec_name] sizes_in = group._var_sizes[vec_name]['input'] sizes_out = group._var_sizes[vec_name]['output'] offsets_in = offsets[vec_name]['input'] offsets_out = offsets[vec_name]['output'] # Loop through all connections owned by this system for abs_in, abs_out in iteritems(group._conn_abs_in2out): if abs_out not in relvars['output']: continue # Only continue if the input exists on this processor if abs_in in abs2meta and abs_in in relvars['input']: # Get meta meta_in = abs2meta[abs_in] meta_out = allprocs_abs2meta[abs_out] idx_in = allprocs_abs2idx[abs_in] idx_out = allprocs_abs2idx[abs_out] # Read in and process src_indices src_indices = meta_in['src_indices'] if src_indices is None: owner = group._owning_rank[abs_out] # if the input is larger than the output on a single proc, we have # to just loop over the procs in the same way we do when src_indices # is defined. if meta_in['size'] > sizes_out[owner, idx_out]: src_indices = np.arange(meta_in['size'], dtype=INT_DTYPE) elif src_indices.ndim == 1: src_indices = convert_neg(src_indices, meta_out['global_size']) else: src_indices = _flatten_src_indices(src_indices, meta_in['shape'], meta_out['global_shape'], meta_out['global_size']) # 1. Compute the output indices if src_indices is None: start = 0 if owner == 0 else np.sum(sizes_out[:owner, idx_out]) offset = offsets_out[owner, idx_out] - start output_inds = np.arange(offset, offset + meta_in['size'], dtype=INT_DTYPE) else: output_inds = np.zeros(src_indices.size, INT_DTYPE) start = end = 0 for iproc in range(group.comm.size): end += sizes_out[iproc, idx_out] # The part of src on iproc on_iproc = np.logical_and(start <= src_indices, src_indices < end) if np.any(on_iproc): # This converts from iproc-then-ivar to ivar-then-iproc ordering # Subtract off part of previous procs # Then add all variables on previous procs # Then all previous variables on this proc # - np.sum(out_sizes[:iproc, idx_out]) # + np.sum(out_sizes[:iproc, :]) # + np.sum(out_sizes[iproc, :idx_out]) # + inds offset = offsets_out[iproc, idx_out] - start output_inds[on_iproc] = src_indices[on_iproc] + offset start = end # 2. Compute the input indices input_inds = np.arange(offsets_in[myproc, idx_in], offsets_in[myproc, idx_in] + sizes_in[myproc, idx_in], dtype=INT_DTYPE) # Now the indices are ready - input_inds, output_inds xfer_in.append(input_inds) xfer_out.append(output_inds) isub = abs2isub[abs_in] fwd_xfer_in[isub].append(input_inds) fwd_xfer_out[isub].append(output_inds) if rev and abs_out in abs2isub: isub = abs2isub[abs_out] rev_xfer_in[isub].append(input_inds) rev_xfer_out[isub].append(output_inds) xfer_in = merge(xfer_in) xfer_out = merge(xfer_out) for isub in range(nsub_allprocs): fwd_xfer_in[isub] = merge(fwd_xfer_in[isub]) fwd_xfer_out[isub] = merge(fwd_xfer_out[isub]) if rev: rev_xfer_in[isub] = merge(rev_xfer_in[isub]) rev_xfer_out[isub] = merge(rev_xfer_out[isub]) out_vec = vectors['output'][vec_name] transfers[vec_name] = {} xfer_all = PETScTransfer(vectors['input'][vec_name], out_vec, xfer_in, xfer_out, group.comm) transfers[vec_name]['fwd', None] = xfer_all if rev: transfers[vec_name]['rev', None] = xfer_all for isub in range(nsub_allprocs): transfers[vec_name]['fwd', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], fwd_xfer_in[isub], fwd_xfer_out[isub], group.comm) if rev: transfers[vec_name]['rev', isub] = PETScTransfer( vectors['input'][vec_name], vectors['output'][vec_name], rev_xfer_in[isub], rev_xfer_out[isub], group.comm) if group._use_derivatives: transfers['nonlinear'] = transfers['linear']