def compare_get_global_values(i, provide_shape): """ Compares the results of get global values for a vector of shape ``(comm.size*2, i)``. :param int i: Dimension of the vector of length ``comm.size*2`` """ if comm.rank == 0: if i == 0: original_array = np.array(np.random.random((comm.size * 2,))) else: original_array = np.array(np.random.random((comm.size * 2, i))) else: original_array = None original_array = comm.bcast(original_array) my_len = original_array.shape[0] / comm.size my_index = range(0 + comm.rank * my_len, (comm.rank + 1) * my_len) if i == 0: my_array = original_array[my_index] else: my_array = original_array[my_index, :] if provide_shape: recomposed_array = util.get_global_values(my_array, original_array.shape) else: recomposed_array = util.get_global_values(my_array) nptest.assert_array_equal(original_array, recomposed_array)
def compare_get_global_values(i, provide_shape): """ Compares the results of get global values for a vector of shape ``(comm.size*2, i)``. :param int i: Dimension of the vector of length ``comm.size*2`` """ if comm.rank == 0: if i == 0: original_array = np.array(np.random.random((comm.size * 2, ))) else: original_array = np.array(np.random.random((comm.size * 2, i))) else: original_array = None original_array = comm.bcast(original_array) my_len = original_array.shape[0] / comm.size my_index = range(0 + comm.rank * my_len, (comm.rank + 1) * my_len) if i == 0: my_array = original_array[my_index] else: my_array = original_array[my_index, :] if provide_shape: recomposed_array = util.get_global_values(my_array, original_array.shape) else: recomposed_array = util.get_global_values(my_array) nptest.assert_array_equal(original_array, recomposed_array)
def find_good_sets(grad_tensor, good_sets_prev, unique_indices, num_optsets_return, cond_tol, volume): r""" #TODO: Use the idea we only know vectors are with 10% accuracy to guide inner_prod tol and condnum_tol. Given gradient vectors at each center in the parameter space and given good sets of size n - 1, return good sets of size n. That is, return sets of size n that have average condition number less than some tolerance. :param grad_tensor: Gradient vectors at each centers in the parameter space :math:'\Lambda' for each QoI map. :type grad_tensor: :class:`np.ndarray` of shape (num_centers,num_qois,Ldim) where num_centers is the number of points in :math:'\Lambda' we have approximated the gradient vectors, num_qois is the total number of possible QoIs to choose from, Ldim is the dimension of :math:`\Lambda`. :param good_sets_prev: Good sets of QoIs of size n - 1. :type good_sets_prev: :class:`np.ndarray` of size (num_good_sets_prev, n - 1) :param unique_indices: Unique QoIs to consider. :type unique_indices: :class:'np.ndarray' of size (num_unique_qois, 1) :param int num_optsets_return: Number of best sets to return :param float cond_tol: Throw out all sets of QoIs with average condition number greater than this. :param boolean volume: If volume is True, use ``calculate_avg_volume`` to determine optimal QoIs :rtype: tuple :returns: (good_sets, best_sets, optsingvals_tensor) where good sets has size (num_good_sets, n), best sets has size (num_optsets_return, n + 1) and optsingvals_tensor has size (num_centers, n, Lambda_dim) """ num_centers = grad_tensor.shape[0] Lambda_dim = grad_tensor.shape[2] num_qois_return = good_sets_prev.shape[1] + 1 comm.Barrier() # Initialize best sets and set all condition numbers large best_sets = np.zeros([num_optsets_return, num_qois_return + 1]) best_sets[:, 0] = np.inf good_sets = np.zeros([1, num_qois_return]) count_qois = 0 optsingvals_tensor = np.zeros( [num_centers, num_qois_return, num_optsets_return]) # For each good set of size n - 1, find the possible sets of size n and # compute the average condition number of each count_qois = 0 for i in range(good_sets_prev.shape[0]): min_ind = np.max(good_sets_prev[i, :]) # Find all possible combinations of QoIs that include this set of n - 1 if comm.rank == 0: inds_notin_set = util.fix_dimensions_vector_2darray(list(set(\ unique_indices) - set(good_sets_prev[i, :]))) # Choose only the QoI indices > min_ind so we do not repeat sets inds_notin_set = util.fix_dimensions_vector_2darray(inds_notin_set[\ inds_notin_set > min_ind]) qoi_combs = util.fix_dimensions_vector_2darray(np.append(np.tile(\ good_sets_prev[i, :], [inds_notin_set.shape[0], 1]), inds_notin_set, axis=1)) qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, compute the average condition number and add the # set to good_sets if it is less than cond_tol for qoi_set in range(len(qoi_combs)): count_qois += 1 curr_set = util.fix_dimensions_vector_2darray(qoi_combs[qoi_set])\ .transpose() if volume == False: (current_condnum, singvals) = calculate_avg_condnum(grad_tensor, qoi_combs[qoi_set]) else: (current_condnum, singvals) = calculate_avg_volume(grad_tensor, qoi_combs[qoi_set]) # If its a good set, add it to good_sets if current_condnum < cond_tol: good_sets = np.append(good_sets, curr_set, axis=0) # If the average condition number is less than the max condition # number in our best_sets, add it to best_sets if current_condnum < best_sets[-1, 0]: best_sets[-1, :] = np.append(np.array([current_condnum]), qoi_combs[qoi_set]) order = best_sets[:, 0].argsort() best_sets = best_sets[order] # Store the corresponding singular values optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and condition numbers from each processor good_sets = comm.gather(good_sets, root=0) best_sets = np.array(comm.gather(best_sets, root=0)) count_qois = np.array(comm.gather(count_qois, root=0)) # Find the num_optsets_return smallest condition numbers from all processors if comm.rank == 0: # Organize the best sets best_sets = best_sets.reshape(num_optsets_return * \ comm.size, num_qois_return + 1) [temp, uniq_inds_best] = np.unique(best_sets[:, 0], return_index=True) best_sets = best_sets[uniq_inds_best, :] best_sets = best_sets[best_sets[:, 0].argsort()] best_sets = best_sets[:num_optsets_return, :] # Organize the good sets good_sets_new = np.zeros([1, num_qois_return]) for each in good_sets: good_sets_new = np.append(good_sets_new, each[1:], axis=0) good_sets = good_sets_new print 'Possible sets of QoIs of size %i : '%good_sets.shape[1],\ np.sum(count_qois) print 'Good sets of QoIs of size %i : '%good_sets.shape[1],\ good_sets.shape[0] - 1 comm.Barrier() best_sets = comm.bcast(best_sets, root=0) good_sets = comm.bcast(good_sets, root=0) return (good_sets[1:].astype(int), best_sets, optsingvals_tensor)
def chooseOptQoIs_verbose(grad_tensor, qoiIndices=None, num_qois_return=None, num_optsets_return=None, inner_prod_tol=1.0, volume=False, remove_zeros=True): r""" Given gradient vectors at some points (centers) in the parameter space, a set of QoIs to choose from, and the number of desired QoIs to return, this method returns the ``num_optsets_return`` best sets of QoIs with with repsect to either the average condition number of the matrix formed by the gradient vectors of each QoI map, or the average volume of the inverse problem us this set of QoIs, computed as the product of the singular values of the same matrix. This method is brute force, i.e., if the method is given 10,000 QoIs and told to return the N best sets of 3, it will check all 10,000 choose 3 possible sets. See chooseOptQoIs_large for a less computationally expensive approach. :param grad_tensor: Gradient vectors at each point of interest in the parameter space :math:`\Lambda` for each QoI map. :type grad_tensor: :class:`np.ndarray` of shape (num_centers, num_qois, Lambda_dim) where num_centers is the number of points in :math:`\Lambda` we have approximated the gradient vectors and num_qois is the total number of possible QoIs to choose from :param qoiIndices: Set of QoIs to consider from grad_tensor. Default is range(0, grad_tensor.shape[1]) :type qoiIndices: :class:`np.ndarray` of size (1, num QoIs to consider) :param int num_qois_return: Number of desired QoIs to use in the inverse problem. Default is Lambda_dim :param int num_optsets_return: Number of best sets to return Default is 10 :param boolean volume: If volume is True, use ``calculate_avg_volume`` to determine optimal QoIs :param boolean remove_zeros: If True, ``find_unique_vecs`` will remove any QoIs that have a zero gradient vector at atleast one point in :math:`\Lambda`. :rtype: tuple :returns: (condnum_indices_mat, optsingvals) where condnum_indices_mat has shape (num_optsets_return, num_qois_return+1) and optsingvals has shape (num_centers, num_qois_return, num_optsets_return) """ num_centers = grad_tensor.shape[0] Lambda_dim = grad_tensor.shape[2] if qoiIndices is None: qoiIndices = range(0, grad_tensor.shape[1]) if num_qois_return is None: num_qois_return = Lambda_dim if num_optsets_return is None: num_optsets_return = 10 qoiIndices = find_unique_vecs(grad_tensor, inner_prod_tol, qoiIndices, remove_zeros) # Find all posible combinations of QoIs if comm.rank == 0: qoi_combs = np.array( list(combinations(list(qoiIndices), num_qois_return))) print 'Possible sets of QoIs : ', qoi_combs.shape[0] qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, check the skewness and keep the sets # that have the best skewness, i.e., smallest condition number condnum_indices_mat = np.zeros([num_optsets_return, num_qois_return + 1]) condnum_indices_mat[:, 0] = np.inf optsingvals_tensor = np.zeros( [num_centers, num_qois_return, num_optsets_return]) for qoi_set in range(len(qoi_combs)): if volume == False: (current_condnum, singvals) = calculate_avg_condnum(grad_tensor, qoi_combs[qoi_set]) else: (current_condnum, singvals) = calculate_avg_volume(grad_tensor, qoi_combs[qoi_set]) if current_condnum < condnum_indices_mat[-1, 0]: condnum_indices_mat[-1, :] = np.append(np.array([current_condnum]), qoi_combs[qoi_set]) order = condnum_indices_mat[:, 0].argsort() condnum_indices_mat = condnum_indices_mat[order] optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and condition numbers from each processor condnum_indices_mat = np.array(comm.gather(condnum_indices_mat, root=0)) optsingvals_tensor = np.array(comm.gather(optsingvals_tensor, root=0)) # Find the num_optsets_return smallest condition numbers from all processors if comm.rank == 0: condnum_indices_mat = condnum_indices_mat.reshape(num_optsets_return * \ comm.size, num_qois_return + 1) optsingvals_tensor = optsingvals_tensor.reshape( num_centers, num_qois_return, num_optsets_return * comm.size) order = condnum_indices_mat[:, 0].argsort() condnum_indices_mat = condnum_indices_mat[order] condnum_indices_mat = condnum_indices_mat[:num_optsets_return, :] optsingvals_tensor = optsingvals_tensor[:, :, order] optsingvals_tensor = optsingvals_tensor[:, :, :num_optsets_return] condnum_indices_mat = comm.bcast(condnum_indices_mat, root=0) optsingvals_tensor = comm.bcast(optsingvals_tensor, root=0) return (condnum_indices_mat, optsingvals_tensor)
def chooseOptQoIs_verbose(grad_tensor, qoiIndices=None, num_qois_return=None, num_optsets_return=None): r""" Given gradient vectors at some points(centers) in the parameter space, a set of QoIs to choose from, and the number of desired QoIs to return, this method return the set of optimal QoIs to use in the inverse problem by choosing the set with optimal skewness properties. Also a tensor that represents the singualre values of the matrices formed by the gradient vectors of the optimal QoIs at each center is returned. :param grad_tensor: Gradient vectors at each point of interest in the parameter space :math:`\Lambda` for each QoI map. :type grad_tensor: :class:`np.ndarray` of shape (num_centers, num_qois, Lambda_dim) where num_centers is the number of points in :math:`\Lambda` we have approximated the gradient vectors and num_qois is the total number of possible QoIs to choose from :param qoiIndices: Set of QoIs to consider from grad_tensor. Default is range(0, grad_tensor.shape[1]) :type qoiIndices: :class:`np.ndarray` of size (1, num QoIs to consider) :param int num_qois_return: Number of desired QoIs to use in the inverse problem. Default is Lambda_dim :param int num_optsets_return: Number of best sets to return Default is 10 :rtype: tuple :returns: (condnum_indices_mat, optsingvals) where condnum_indices_mat has shape (num_optsets_return, num_qois_return+1) and optsingvals has shape (num_centers, num_qois_return, num_optsets_return) """ num_centers = grad_tensor.shape[0] Lambda_dim = grad_tensor.shape[2] if qoiIndices is None: qoiIndices = range(0, grad_tensor.shape[1]) if num_qois_return is None: num_qois_return = Lambda_dim if num_optsets_return is None: num_optsets_return = 10 # Find all posible combinations of QoIs if comm.rank == 0: qoi_combs = np.array(list(combinations(list(qoiIndices), num_qois_return))) print "Possible sets of QoIs : ", qoi_combs.shape[0] qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, check the skewness and keep the sets # that have the best skewness, i.e., smallest condition number condnum_indices_mat = np.zeros([num_optsets_return, num_qois_return + 1]) condnum_indices_mat[:, 0] = 1e11 optsingvals_tensor = np.zeros([num_centers, num_qois_return, num_optsets_return]) for qoi_set in range(len(qoi_combs)): singvals = np.linalg.svd(grad_tensor[:, qoi_combs[qoi_set], :], compute_uv=False) # Find the centers that have atleast one zero sinular value indz = singvals[:, -1] == 0 indnz = singvals[:, -1] != 0 current_condnum = ( np.sum(singvals[indnz, 0] / singvals[indnz, -1], axis=0) + 1e9 * np.sum(indz) ) / singvals.shape[0] if current_condnum < condnum_indices_mat[-1, 0]: condnum_indices_mat[-1, :] = np.append(np.array([current_condnum]), qoi_combs[qoi_set]) order = condnum_indices_mat[:, 0].argsort() condnum_indices_mat = condnum_indices_mat[order] optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and condition numbers from each processor condnum_indices_mat = np.array(comm.gather(condnum_indices_mat, root=0)) optsingvals_tensor = np.array(comm.gather(optsingvals_tensor, root=0)) # Find the num_optsets_return smallest condition numbers from all processors if comm.rank == 0: condnum_indices_mat = condnum_indices_mat.reshape(num_optsets_return * comm.size, num_qois_return + 1) optsingvals_tensor = optsingvals_tensor.reshape(num_centers, num_qois_return, num_optsets_return * comm.size) order = condnum_indices_mat[:, 0].argsort() condnum_indices_mat = condnum_indices_mat[order] condnum_indices_mat = condnum_indices_mat[:num_optsets_return, :] optsingvals_tensor = optsingvals_tensor[:, :, order] optsingvals_tensor = optsingvals_tensor[:, :, :num_optsets_return] condnum_indices_mat = comm.bcast(condnum_indices_mat, root=0) optsingvals_tensor = comm.bcast(optsingvals_tensor, root=0) return (condnum_indices_mat, optsingvals_tensor)
def find_good_sets(input_set, good_sets_prev, unique_indices, num_optsets_return, measskew_tol, measure): r""" .. todo:: Use the idea we only know vectors are with 10% accuracy to guide inner_prod tol and skewness_tol. Given gradient vectors at each center in the parameter space and given good sets of size (n - 1), return good sets of size n. That is, return sets of size n that have average measure(skewness) less than some tolerance. :param input_set: The input sample set. Make sure the attribute _jacobians is not None. :type input_set: :class:`~bet.sample.sample_set` :param good_sets_prev: Good sets of QoIs of size n - 1. :type good_sets_prev: :class:`np.ndarray` of size (num_good_sets_prev, n - 1) :param unique_indices: Unique QoIs to consider. :type unique_indices: :class:`np.ndarray` of size (num_unique_qois, 1) :param int num_optsets_return: Number of best sets to return :param float measskew_tol: Throw out all sets of QoIs with average measure(skewness) number greater than this. :param boolean measure: If measure is True, use ``calculate_avg_measure`` to determine optimal QoIs, else use ``calculate_avg_skewness`` :rtype: tuple :returns: (good_sets, best_sets, optsingvals_tensor) where good sets has size (num_good_sets, n), best sets has size (num_optsets_return, n + 1) and optsingvals_tensor has size (num_centers, n, input_dim) """ if input_set._jacobians is None: raise ValueError("You must have jacobians to use this method.") num_centers = input_set._jacobians.shape[0] num_qois_return = good_sets_prev.shape[1] + 1 comm.Barrier() # Initialize best sets and set all skewness values large best_sets = np.zeros([num_optsets_return, num_qois_return + 1]) best_sets[:, 0] = np.inf good_sets = np.zeros([1, num_qois_return]) count_qois = 0 optsingvals_tensor = np.zeros( [num_centers, num_qois_return, num_optsets_return]) # For each good set of size (n - 1), find the possible sets of size n and # compute the average skewness of each count_qois = 0 for i in xrange(good_sets_prev.shape[0]): min_ind = np.max(good_sets_prev[i, :]) # Find all possible combinations of QoIs that include this set of # (n - 1) if comm.rank == 0: inds_notin_set = util.fix_dimensions_vector_2darray(list(set(\ unique_indices) - set(good_sets_prev[i, :]))) # Choose only the QoI indices > min_ind so we do not repeat sets inds_notin_set = util.fix_dimensions_vector_2darray(inds_notin_set[\ inds_notin_set > min_ind]) qoi_combs = util.fix_dimensions_vector_2darray(np.append(np.tile(\ good_sets_prev[i, :], [inds_notin_set.shape[0], 1]), inds_notin_set, axis=1)) qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, compute the average measure(skewness) and add # the set to good_sets if it is less than measskew_tol for qoi_set in xrange(len(qoi_combs)): count_qois += 1 curr_set = util.fix_dimensions_vector_2darray(qoi_combs[qoi_set])\ .transpose() if measure is False: (current_measskew, singvals) = calculate_avg_skewness(input_set, qoi_combs[qoi_set]) else: (current_measskew, singvals) = calculate_avg_measure(input_set, qoi_combs[qoi_set]) # If its a good set, add it to good_sets if current_measskew < measskew_tol: good_sets = np.append(good_sets, curr_set, axis=0) # If the average skewness is less than the maxskewness # in our best_sets, add it to best_sets if current_measskew < best_sets[-1, 0]: best_sets[-1, :] = np.append(np.array([current_measskew]), qoi_combs[qoi_set]) order = best_sets[:, 0].argsort() best_sets = best_sets[order] # Store the corresponding singular values optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and skewness values from each processor good_sets = comm.gather(good_sets, root=0) best_sets = np.array(comm.gather(best_sets, root=0)) count_qois = np.array(comm.gather(count_qois, root=0)) # Find the num_optsets_return smallest skewness from all processors if comm.rank == 0: # Organize the best sets best_sets = best_sets.reshape(num_optsets_return * \ comm.size, num_qois_return + 1) [_, uniq_inds_best] = np.unique(best_sets[:, 0], return_index=True) best_sets = best_sets[uniq_inds_best, :] best_sets = best_sets[best_sets[:, 0].argsort()] best_sets = best_sets[:num_optsets_return, :] # Organize the good sets good_sets_new = np.zeros([1, num_qois_return]) for each in good_sets: good_sets_new = np.append(good_sets_new, each[1:], axis=0) good_sets = good_sets_new logging.info('Possible sets of QoIs of size {} : {}'.format(\ good_sets.shape[1], np.sum(count_qois))) logging.info('Good sets of QoIs of size {} : {}'.format(\ good_sets.shape[1], good_sets.shape[0] - 1)) comm.Barrier() best_sets = comm.bcast(best_sets, root=0) good_sets = comm.bcast(good_sets, root=0) return (good_sets[1:].astype(int), best_sets, optsingvals_tensor)
def chooseOptQoIs_verbose(input_set, qoiIndices=None, num_qois_return=None, num_optsets_return=None, inner_prod_tol=1.0, measure=False, remove_zeros=True): r""" Given gradient vectors at some points (centers) in the parameter space, a set of QoIs to choose from, and the number of desired QoIs to return, this method returns the ``num_optsets_return`` best sets of QoIs with with repsect to either the average measure of the matrix formed by the gradient vectors of each QoI map, OR the average skewness of the inverse image of this set of QoIs, computed as the product of the singular values of the same matrix. This method is brute force, i.e., if the method is given 10,000 QoIs and told to return the N best sets of 3, it will check all 10,000 choose 3 possible sets. See chooseOptQoIs_large for a less computationally expensive approach. :param input_set: The input sample set. Make sure the attribute _jacobians is not None :type input_set: :class:`~bet.sample.sample_set` :param qoiIndices: Set of QoIs to consider. Default is xrange(0, input_set._jacobians.shape[1]) :type qoiIndices: :class:`np.ndarray` of size (1, num QoIs to consider) :param int num_qois_return: Number of desired QoIs to use in the inverse problem. Default is input_dim :param int num_optsets_return: Number of best sets to return Default is 10 :param boolean measure: If measure is True, use ``calculate_avg_measure`` to determine optimal QoIs, else use ``calculate_avg_skewness`` :param boolean remove_zeros: If True, ``find_unique_vecs`` will remove any QoIs that have a zero gradient :rtype: `np.ndarray` of shape (num_optsets_returned, num_qois_returned + 1) :returns: measure_skewness_indices_mat """ G = input_set._jacobians if G is None: raise ValueError("You must have jacobians to use this method.") input_dim = input_set._dim num_centers = G.shape[0] if qoiIndices is None: qoiIndices = xrange(0, G.shape[1]) if num_qois_return is None: num_qois_return = input_dim if num_optsets_return is None: num_optsets_return = 10 # Remove QoIs that have zero gradients at any of the centers qoiIndices = find_unique_vecs(input_set, inner_prod_tol, qoiIndices, remove_zeros) # Find all posible combinations of QoIs if comm.rank == 0: qoi_combs = np.array( list(combinations(list(qoiIndices), num_qois_return))) logging.info('Possible sets of QoIs : {}'.format(qoi_combs.shape[0])) qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, check the skewness and keep the sets # that have the smallest skewness measure_skewness_indices_mat = np.zeros( [num_optsets_return, num_qois_return + 1]) measure_skewness_indices_mat[:, 0] = np.inf optsingvals_tensor = np.zeros( [num_centers, num_qois_return, num_optsets_return]) for qoi_set in xrange(len(qoi_combs)): if measure == False: (current_measskew, singvals) = calculate_avg_skewness(input_set, qoi_combs[qoi_set]) else: (current_measskew, singvals) = calculate_avg_measure(input_set, qoi_combs[qoi_set]) if current_measskew < measure_skewness_indices_mat[-1, 0]: measure_skewness_indices_mat[-1, :] = np.append(np.array(\ [current_measskew]), qoi_combs[qoi_set]) order = measure_skewness_indices_mat[:, 0].argsort() measure_skewness_indices_mat = measure_skewness_indices_mat[order] optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and skewness values from each processor measure_skewness_indices_mat = np.array(comm.gather(\ measure_skewness_indices_mat, root=0)) optsingvals_tensor = np.array(comm.gather(optsingvals_tensor, root=0)) # Find the num_optsets_return smallest skewness values from all processors if comm.rank == 0: measure_skewness_indices_mat = measure_skewness_indices_mat.reshape(\ num_optsets_return * comm.size, num_qois_return + 1) optsingvals_tensor = optsingvals_tensor.reshape( num_centers, num_qois_return, num_optsets_return * comm.size) order = measure_skewness_indices_mat[:, 0].argsort() measure_skewness_indices_mat = measure_skewness_indices_mat[order] measure_skewness_indices_mat = measure_skewness_indices_mat[\ :num_optsets_return, :] optsingvals_tensor = optsingvals_tensor[:, :, order] optsingvals_tensor = optsingvals_tensor[:, :, :num_optsets_return] measure_skewness_indices_mat = comm.bcast(measure_skewness_indices_mat, root=0) optsingvals_tensor = comm.bcast(optsingvals_tensor, root=0) return (measure_skewness_indices_mat, optsingvals_tensor)
def find_good_sets(grad_tensor, good_sets_prev, unique_indices, num_optsets_return, cond_tol, volume): r""" #TODO: Use the idea we only know vectors are with 10% accuracy to guide inner_prod tol and condnum_tol. Given gradient vectors at each center in the parameter space and given good sets of size n - 1, return good sets of size n. That is, return sets of size n that have average condition number less than some tolerance. :param grad_tensor: Gradient vectors at each centers in the parameter space :math:'\Lambda' for each QoI map. :type grad_tensor: :class:`np.ndarray` of shape (num_centers,num_qois,Ldim) where num_centers is the number of points in :math:'\Lambda' we have approximated the gradient vectors, num_qois is the total number of possible QoIs to choose from, Ldim is the dimension of :math:`\Lambda`. :param good_sets_prev: Good sets of QoIs of size n - 1. :type good_sets_prev: :class:`np.ndarray` of size (num_good_sets_prev, n - 1) :param unique_indices: Unique QoIs to consider. :type unique_indices: :class:'np.ndarray' of size (num_unique_qois, 1) :param int num_optsets_return: Number of best sets to return :param float cond_tol: Throw out all sets of QoIs with average condition number greater than this. :param boolean volume: If volume is True, use ``calculate_avg_volume`` to determine optimal QoIs :rtype: tuple :returns: (good_sets, best_sets, optsingvals_tensor) where good sets has size (num_good_sets, n), best sets has size (num_optsets_return, n + 1) and optsingvals_tensor has size (num_centers, n, Lambda_dim) """ num_centers = grad_tensor.shape[0] Lambda_dim = grad_tensor.shape[2] num_qois_return = good_sets_prev.shape[1] + 1 comm.Barrier() # Initialize best sets and set all condition numbers large best_sets = np.zeros([num_optsets_return, num_qois_return + 1]) best_sets[:, 0] = 1E99 good_sets = np.zeros([1, num_qois_return]) count_qois = 0 optsingvals_tensor = np.zeros([num_centers, num_qois_return, num_optsets_return]) # For each good set of size n - 1, find the possible sets of size n and # compute the average condition number of each count_qois = 0 for i in range(good_sets_prev.shape[0]): min_ind = np.max(good_sets_prev[i, :]) # Find all possible combinations of QoIs that include this set of n - 1 if comm.rank == 0: inds_notin_set = util.fix_dimensions_vector_2darray(list(set(\ unique_indices) - set(good_sets_prev[i, :]))) # Choose only the QoI indices > min_ind so we do not repeat sets inds_notin_set = util.fix_dimensions_vector_2darray(inds_notin_set[\ inds_notin_set > min_ind]) qoi_combs = util.fix_dimensions_vector_2darray(np.append(np.tile(\ good_sets_prev[i, :], [inds_notin_set.shape[0], 1]), inds_notin_set, axis=1)) qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, compute the average condition number and add the # set to good_sets if it is less than cond_tol for qoi_set in range(len(qoi_combs)): count_qois += 1 curr_set = util.fix_dimensions_vector_2darray(qoi_combs[qoi_set])\ .transpose() if volume == False: (current_condnum, singvals) = calculate_avg_condnum(grad_tensor, qoi_combs[qoi_set]) else: (current_condnum, singvals) = calculate_avg_volume(grad_tensor, qoi_combs[qoi_set]) # If its a good set, add it to good_sets if current_condnum < cond_tol: good_sets = np.append(good_sets, curr_set, axis=0) # If the average condition number is less than the max condition # number in our best_sets, add it to best_sets if current_condnum < best_sets[-1, 0]: best_sets[-1, :] = np.append(np.array([current_condnum]), qoi_combs[qoi_set]) order = best_sets[:, 0].argsort() best_sets = best_sets[order] # Store the corresponding singular values optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and condition numbers from each processor good_sets = comm.gather(good_sets, root=0) best_sets = np.array(comm.gather(best_sets, root=0)) count_qois = np.array(comm.gather(count_qois, root=0)) # Find the num_optsets_return smallest condition numbers from all processors if comm.rank == 0: # Organize the best sets best_sets = best_sets.reshape(num_optsets_return * \ comm.size, num_qois_return + 1) [temp, uniq_inds_best] = np.unique(best_sets[:, 0], return_index=True) best_sets = best_sets[uniq_inds_best, :] best_sets = best_sets[best_sets[:, 0].argsort()] best_sets = best_sets[:num_optsets_return, :] # Organize the good sets good_sets_new = np.zeros([1, num_qois_return]) for each in good_sets: good_sets_new = np.append(good_sets_new, each[1:], axis=0) good_sets = good_sets_new print 'Possible sets of QoIs of size %i : '%good_sets.shape[1],\ np.sum(count_qois) print 'Good sets of QoIs of size %i : '%good_sets.shape[1],\ good_sets.shape[0] - 1 comm.Barrier() best_sets = comm.bcast(best_sets, root=0) good_sets = comm.bcast(good_sets, root=0) return (good_sets[1:].astype(int), best_sets, optsingvals_tensor)
def chooseOptQoIs_verbose(grad_tensor, qoiIndices=None, num_qois_return=None, num_optsets_return=None, inner_prod_tol=1.0, volume=False, remove_zeros=True): r""" Given gradient vectors at some points (centers) in the parameter space, a set of QoIs to choose from, and the number of desired QoIs to return, this method returns the ``num_optsets_return`` best sets of QoIs with with repsect to either the average condition number of the matrix formed by the gradient vectors of each QoI map, or the average volume of the inverse problem us this set of QoIs, computed as the product of the singular values of the same matrix. This method is brute force, i.e., if the method is given 10,000 QoIs and told to return the N best sets of 3, it will check all 10,000 choose 3 possible sets. See chooseOptQoIs_large for a less computationally expensive approach. :param grad_tensor: Gradient vectors at each point of interest in the parameter space :math:`\Lambda` for each QoI map. :type grad_tensor: :class:`np.ndarray` of shape (num_centers, num_qois, Lambda_dim) where num_centers is the number of points in :math:`\Lambda` we have approximated the gradient vectors and num_qois is the total number of possible QoIs to choose from :param qoiIndices: Set of QoIs to consider from grad_tensor. Default is range(0, grad_tensor.shape[1]) :type qoiIndices: :class:`np.ndarray` of size (1, num QoIs to consider) :param int num_qois_return: Number of desired QoIs to use in the inverse problem. Default is Lambda_dim :param int num_optsets_return: Number of best sets to return Default is 10 :param boolean volume: If volume is True, use ``calculate_avg_volume`` to determine optimal QoIs :param boolean remove_zeros: If True, ``find_unique_vecs`` will remove any QoIs that have a zero gradient vector at atleast one point in :math:`\Lambda`. :rtype: tuple :returns: (condnum_indices_mat, optsingvals) where condnum_indices_mat has shape (num_optsets_return, num_qois_return+1) and optsingvals has shape (num_centers, num_qois_return, num_optsets_return) """ num_centers = grad_tensor.shape[0] Lambda_dim = grad_tensor.shape[2] if qoiIndices is None: qoiIndices = range(0, grad_tensor.shape[1]) if num_qois_return is None: num_qois_return = Lambda_dim if num_optsets_return is None: num_optsets_return = 10 qoiIndices = find_unique_vecs(grad_tensor, inner_prod_tol, qoiIndices, remove_zeros) # Find all posible combinations of QoIs if comm.rank == 0: qoi_combs = np.array(list(combinations(list(qoiIndices), num_qois_return))) print 'Possible sets of QoIs : ', qoi_combs.shape[0] qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, check the skewness and keep the sets # that have the best skewness, i.e., smallest condition number condnum_indices_mat = np.zeros([num_optsets_return, num_qois_return + 1]) condnum_indices_mat[:, 0] = 1E99 optsingvals_tensor = np.zeros([num_centers, num_qois_return, num_optsets_return]) for qoi_set in range(len(qoi_combs)): if volume == False: (current_condnum, singvals) = calculate_avg_condnum(grad_tensor, qoi_combs[qoi_set]) else: (current_condnum, singvals) = calculate_avg_volume(grad_tensor, qoi_combs[qoi_set]) if current_condnum < condnum_indices_mat[-1, 0]: condnum_indices_mat[-1, :] = np.append(np.array([current_condnum]), qoi_combs[qoi_set]) order = condnum_indices_mat[:, 0].argsort() condnum_indices_mat = condnum_indices_mat[order] optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and condition numbers from each processor condnum_indices_mat = np.array(comm.gather(condnum_indices_mat, root=0)) optsingvals_tensor = np.array(comm.gather(optsingvals_tensor, root=0)) # Find the num_optsets_return smallest condition numbers from all processors if comm.rank == 0: condnum_indices_mat = condnum_indices_mat.reshape(num_optsets_return * \ comm.size, num_qois_return + 1) optsingvals_tensor = optsingvals_tensor.reshape(num_centers, num_qois_return, num_optsets_return * comm.size) order = condnum_indices_mat[:, 0].argsort() condnum_indices_mat = condnum_indices_mat[order] condnum_indices_mat = condnum_indices_mat[:num_optsets_return, :] optsingvals_tensor = optsingvals_tensor[:, :, order] optsingvals_tensor = optsingvals_tensor[:, :, :num_optsets_return] condnum_indices_mat = comm.bcast(condnum_indices_mat, root=0) optsingvals_tensor = comm.bcast(optsingvals_tensor, root=0) return (condnum_indices_mat, optsingvals_tensor)
def find_good_sets(input_set, good_sets_prev, unique_indices, num_optsets_return, measskew_tol, measure): r""" .. todo:: Use the idea we only know vectors are with 10% accuracy to guide inner_prod tol and skewness_tol. Given gradient vectors at each center in the parameter space and given good sets of size (n - 1), return good sets of size n. That is, return sets of size n that have average measure(skewness) less than some tolerance. :param input_set: The input sample set. Make sure the attribute _jacobians is not None. :type input_set: :class:`~bet.sample.sample_set` :param good_sets_prev: Good sets of QoIs of size n - 1. :type good_sets_prev: :class:`np.ndarray` of size (num_good_sets_prev, n - 1) :param unique_indices: Unique QoIs to consider. :type unique_indices: :class:`np.ndarray` of size (num_unique_qois, 1) :param int num_optsets_return: Number of best sets to return :param float measskew_tol: Throw out all sets of QoIs with average measure(skewness) number greater than this. :param boolean measure: If measure is True, use ``calculate_avg_measure`` to determine optimal QoIs, else use ``calculate_avg_skewness`` :rtype: tuple :returns: (good_sets, best_sets, optsingvals_tensor) where good sets has size (num_good_sets, n), best sets has size (num_optsets_return, n + 1) and optsingvals_tensor has size (num_centers, n, input_dim) """ if input_set._jacobians is None: raise ValueError("You must have jacobians to use this method.") num_centers = input_set._jacobians.shape[0] num_qois_return = good_sets_prev.shape[1] + 1 comm.Barrier() # Initialize best sets and set all skewness values large best_sets = np.zeros([num_optsets_return, num_qois_return + 1]) best_sets[:, 0] = np.inf good_sets = np.zeros([1, num_qois_return]) count_qois = 0 optsingvals_tensor = np.zeros([num_centers, num_qois_return, num_optsets_return]) # For each good set of size (n - 1), find the possible sets of size n and # compute the average skewness of each count_qois = 0 for i in range(good_sets_prev.shape[0]): min_ind = np.max(good_sets_prev[i, :]) # Find all possible combinations of QoIs that include this set of # (n - 1) if comm.rank == 0: inds_notin_set = util.fix_dimensions_vector_2darray(list(set( unique_indices) - set(good_sets_prev[i, :]))) # Choose only the QoI indices > min_ind so we do not repeat sets inds_notin_set = util.fix_dimensions_vector_2darray(inds_notin_set[ inds_notin_set > min_ind]) qoi_combs = util.fix_dimensions_vector_2darray(np.append(np.tile( good_sets_prev[i, :], [inds_notin_set.shape[0], 1]), inds_notin_set, axis=1)) qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, compute the average measure(skewness) and add # the set to good_sets if it is less than measskew_tol for qoi_set in range(len(qoi_combs)): count_qois += 1 curr_set = util.fix_dimensions_vector_2darray(qoi_combs[qoi_set])\ .transpose() if measure is False: (current_measskew, singvals) = calculate_avg_skewness(input_set, qoi_combs[qoi_set]) else: (current_measskew, singvals) = calculate_avg_measure(input_set, qoi_combs[qoi_set]) # If its a good set, add it to good_sets if current_measskew < measskew_tol: good_sets = np.append(good_sets, curr_set, axis=0) # If the average skewness is less than the maxskewness # in our best_sets, add it to best_sets if current_measskew < best_sets[-1, 0]: best_sets[-1, :] = np.append(np.array([current_measskew]), qoi_combs[qoi_set]) order = best_sets[:, 0].argsort() best_sets = best_sets[order] # Store the corresponding singular values optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and skewness values from each processor good_sets = comm.gather(good_sets, root=0) best_sets = np.array(comm.gather(best_sets, root=0)) count_qois = np.array(comm.gather(count_qois, root=0)) # Find the num_optsets_return smallest skewness from all processors if comm.rank == 0: # Organize the best sets best_sets = best_sets.reshape(num_optsets_return * comm.size, num_qois_return + 1) [_, uniq_inds_best] = np.unique(best_sets[:, 0], return_index=True) best_sets = best_sets[uniq_inds_best, :] best_sets = best_sets[best_sets[:, 0].argsort()] best_sets = best_sets[:num_optsets_return, :] # Organize the good sets good_sets_new = np.zeros([1, num_qois_return]) for each in good_sets: good_sets_new = np.append(good_sets_new, each[1:], axis=0) good_sets = good_sets_new logging.info('Possible sets of QoIs of size {} : {}'.format( good_sets.shape[1], np.sum(count_qois))) logging.info('Good sets of QoIs of size {} : {}'.format( good_sets.shape[1], good_sets.shape[0] - 1)) comm.Barrier() best_sets = comm.bcast(best_sets, root=0) good_sets = comm.bcast(good_sets, root=0) return (good_sets[1:].astype(int), best_sets, optsingvals_tensor)
def chooseOptQoIs_verbose(input_set, qoiIndices=None, num_qois_return=None, num_optsets_return=None, inner_prod_tol=1.0, measure=False, remove_zeros=True): r""" Given gradient vectors at some points (centers) in the parameter space, a set of QoIs to choose from, and the number of desired QoIs to return, this method returns the ``num_optsets_return`` best sets of QoIs with with repsect to either the average measure of the matrix formed by the gradient vectors of each QoI map, OR the average skewness of the inverse image of this set of QoIs, computed as the product of the singular values of the same matrix. This method is brute force, i.e., if the method is given 10,000 QoIs and told to return the N best sets of 3, it will check all 10,000 choose 3 possible sets. See chooseOptQoIs_large for a less computationally expensive approach. :param input_set: The input sample set. Make sure the attribute _jacobians is not None :type input_set: :class:`~bet.sample.sample_set` :param qoiIndices: Set of QoIs to consider. Default is xrange(0, input_set._jacobians.shape[1]) :type qoiIndices: :class:`np.ndarray` of size (1, num QoIs to consider) :param int num_qois_return: Number of desired QoIs to use in the inverse problem. Default is input_dim :param int num_optsets_return: Number of best sets to return Default is 10 :param boolean measure: If measure is True, use ``calculate_avg_measure`` to determine optimal QoIs, else use ``calculate_avg_skewness`` :param boolean remove_zeros: If True, ``find_unique_vecs`` will remove any QoIs that have a zero gradient :rtype: `np.ndarray` of shape (num_optsets_returned, num_qois_returned + 1) :returns: measure_skewness_indices_mat """ G = input_set._jacobians if G is None: raise ValueError("You must have jacobians to use this method.") input_dim = input_set._dim num_centers = G.shape[0] if qoiIndices is None: qoiIndices = range(0, G.shape[1]) if num_qois_return is None: num_qois_return = input_dim if num_optsets_return is None: num_optsets_return = 10 # Remove QoIs that have zero gradients at any of the centers qoiIndices = find_unique_vecs(input_set, inner_prod_tol, qoiIndices, remove_zeros) # Find all posible combinations of QoIs if comm.rank == 0: qoi_combs = np.array(list(combinations(list(qoiIndices), num_qois_return))) logging.info('Possible sets of QoIs : {}'.format(qoi_combs.shape[0])) qoi_combs = np.array_split(qoi_combs, comm.size) else: qoi_combs = None # Scatter them throughout the processors qoi_combs = comm.scatter(qoi_combs, root=0) # For each combination, check the skewness and keep the sets # that have the smallest skewness measure_skewness_indices_mat = np.zeros([num_optsets_return, num_qois_return + 1]) measure_skewness_indices_mat[:, 0] = np.inf optsingvals_tensor = np.zeros([num_centers, num_qois_return, num_optsets_return]) for qoi_set in range(len(qoi_combs)): if measure == False: (current_measskew, singvals) = calculate_avg_skewness(input_set, qoi_combs[qoi_set]) else: (current_measskew, singvals) = calculate_avg_measure(input_set, qoi_combs[qoi_set]) if current_measskew < measure_skewness_indices_mat[-1, 0]: measure_skewness_indices_mat[-1, :] = np.append(np.array( [current_measskew]), qoi_combs[qoi_set]) order = measure_skewness_indices_mat[:, 0].argsort() measure_skewness_indices_mat = measure_skewness_indices_mat[order] optsingvals_tensor[:, :, -1] = singvals optsingvals_tensor = optsingvals_tensor[:, :, order] # Wait for all processes to get to this point comm.Barrier() # Gather the best sets and skewness values from each processor measure_skewness_indices_mat = np.array(comm.gather( measure_skewness_indices_mat, root=0)) optsingvals_tensor = np.array(comm.gather(optsingvals_tensor, root=0)) # Find the num_optsets_return smallest skewness values from all processors if comm.rank == 0: measure_skewness_indices_mat = measure_skewness_indices_mat.reshape( num_optsets_return * comm.size, num_qois_return + 1) optsingvals_tensor = optsingvals_tensor.reshape(num_centers, num_qois_return, num_optsets_return * comm.size) order = measure_skewness_indices_mat[:, 0].argsort() measure_skewness_indices_mat = measure_skewness_indices_mat[order] measure_skewness_indices_mat = measure_skewness_indices_mat[ :num_optsets_return, :] optsingvals_tensor = optsingvals_tensor[:, :, order] optsingvals_tensor = optsingvals_tensor[:, :, :num_optsets_return] measure_skewness_indices_mat = comm.bcast(measure_skewness_indices_mat, root=0) optsingvals_tensor = comm.bcast(optsingvals_tensor, root=0) return (measure_skewness_indices_mat, optsingvals_tensor)