internal_ids = [] for metab in network.metabolites: if not metab.is_external: id_ind = [ind for ind, id in enumerate(ids) if id == metab.id] if len(id_ind): internal_ids.append(id_ind[0]) ids = list(np.delete(ids, internal_ids)) cone = np.delete(cone, internal_ids, axis=1) if mpi_wrapper.is_first_process(): try: np.savetxt(args.out_path, cone, delimiter=',', header=','.join(ids), comments='') except OverflowError: normalised = np.transpose(normalize_columns(np.transpose(cone))) np.savetxt(args.out_path, normalised, delimiter=',', header=','.join(ids), comments='') if args.print_conversions: print_ecms_direct(np.transpose(cone), ids) end = time() mp_print('Ran in %f seconds' % (end - start))
def geometric_ray_adjacency(ray_matrix, plus=[-1], minus=[-1], tol=1e-3, verbose=True, remove_cycles=True): """ Returns r by r adjacency matrix of rays, given ray matrix R. Diagonal is 0, not 1. Calculated using LP adjacency test :param R: ray matrix (columns are generating rays) plus: indices of 'plus' columns minus: indices of 'minus' columns if plus/minus are not provided, find all adjacencies; otherwise only between each + and - pair :return: r by r adjacency matrix """ start = time() matrix_normalized = normalize_columns(ray_matrix) matrix_indep_rows = independent_rows_qr(matrix_normalized) # matrix_indep_rows = independent_rows(matrix_normalized) # set default plus and minus if len(plus) > 0 and plus[0] == -1: plus = [x for x in range(matrix_indep_rows.shape[1])] if len(minus) > 0 and minus[0] == -1: minus = [x for x in range(matrix_indep_rows.shape[1])] mpi_size = get_process_size() mpi_rank = get_process_rank() adjacency = [] nr_tests = len(plus) * len(minus) # first find any column basis of R_indep start_basis = get_basis_columns_qr(matrix_indep_rows) start_basis_inv = np.linalg.inv(matrix_indep_rows[:, start_basis]) for i, p in enumerate(plus): # add the plus ray into the basis plus_basis = add_first_ray(matrix_indep_rows, start_basis_inv, start_basis, p) plus_basis_inv = np.linalg.inv(matrix_indep_rows[:, plus_basis]) for j, m in enumerate(minus): it = j + len(minus) * i if it % mpi_size == mpi_rank: # add the minus ray into the basis plus_minus_basis = add_second_ray(matrix_indep_rows, plus_basis_inv, plus_basis, p, m) res = determine_adjacency(matrix_indep_rows, p, m, plus_minus_basis) if res == 1: adjacency.append((p, m)) if it % (100 * mpi_size) == mpi_rank: mp_print("Process %d is on adjacency test %d of %d (%f %%)" % (mpi_rank, it, nr_tests, it / nr_tests * 100), PRINT_IF_RANK_NONZERO=True) # bases = get_bases(matrix_indep_rows, plus, minus) # for i in range(mpi_rank, nr_tests, mpi_size): # plus_index = i // len(minus) # minus_index = i % len(minus) # basis = bases[plus_index, minus_index] # res = determine_adjacency(matrix_indep_rows, plus[plus_index], minus[minus_index], basis) # if res == 1: # adjacency.append((plus[plus_index], minus[minus_index])) # if i % 100 == 0: # mp_print("Process %d is now on adjacency test %d" % (mpi_rank, i)) # MPI communication step adj_sets = world_allgather(adjacency) for i in range(mpi_size): if i != mpi_rank: adjacency.extend(adj_sets[i]) adjacency.sort() end = time() mp_print("Did LPs in %f seconds" % (end - start)) return adjacency
def drop_redundant_rays(ray_matrix, verbose=True, use_pre_filter=False, rays_are_unique=True, linearities=False, normalised=True): """ :param ray_matrix: :param verbose: :param use_pre_filter: Sometimes, use_pre_filter=True can speed up the calculations, but mostly it doesn't :param rays_are_unique: Boolean that states whether rays given as input are already unique :param linearities: Boolean indicating if linearities are still present :param normalised: Boolean indicating if ray_matrix columns are already normalised :return: """ extreme_inds = [] non_extreme_inds = [] if not rays_are_unique: # First make sure that no duplicate rays are in the matrix ray_matrix = np.transpose( unique(np.transpose(normalize_columns_fraction(ray_matrix)))) # Find 'cycles': combinations of columns of matrix_indep_rows that add up to zero, and remove them if linearities: if verbose: mp_print('Detecting linearities in H_ineq.') ray_matrix, cycle_rays = remove_cycles_redund(ray_matrix) else: cycle_rays = np.zeros((ray_matrix.shape[0], 0)) if not normalised: if verbose: mp_print('Normalizing columns.') matrix_normalized = normalize_columns(ray_matrix) else: matrix_normalized = np.array(ray_matrix, dtype='float') if verbose: mp_print('Selecting independent rows.') matrix_indep_rows = independent_rows_qr(matrix_normalized) if use_pre_filter: filtered_inds = pre_redund(matrix_indep_rows) else: filtered_inds = np.arange(matrix_indep_rows.shape[1]) mpi_size = mpi_wrapper.get_process_size() mpi_rank = mpi_wrapper.get_process_rank() matrix_indep_rows = matrix_indep_rows[:, filtered_inds] # then find any column basis of R_indep start_basis = get_basis_columns_qr(matrix_indep_rows) start_basis_inv = np.linalg.inv(matrix_indep_rows[:, start_basis]) number_rays = matrix_indep_rows.shape[1] for i in range(number_rays): if i % mpi_size == mpi_rank: if i % (10 * mpi_size) == mpi_rank: mp_print( "Process %d is on redundancy test %d of %d (%f %%). Found %d redundant rays." % (mpi_rank, i, number_rays, i / number_rays * 100, len(non_extreme_inds)), PRINT_IF_RANK_NONZERO=True) basis = add_first_ray(matrix_indep_rows, start_basis_inv, start_basis, i) extreme = check_extreme(matrix_indep_rows, i, basis) if extreme: extreme_inds.append(filtered_inds[i]) else: non_extreme_inds.append(filtered_inds[i]) # MPI communication step extreme_sets = mpi_wrapper.world_allgather(extreme_inds) for i in range(mpi_size): if i != mpi_rank: extreme_inds.extend(extreme_sets[i]) extreme_inds.sort() return extreme_inds, cycle_rays
def remove_cycles(R, network, tol=1e-12, verbose=True): """Detect whether there are cycles, by doing an LP. If LP is unbounded find a minimal cycle. Cancel one metabolite with the cycle.""" external_cycles = [] count_since_last_redund = 0 A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr(normalize_columns(R)), only_eq=True) # A_eqtest, b_eqtest, ctest, x0test = setup_cycle_LP(independent_rows(normalize_columns(R)), only_eq=True) basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float')) b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10) cycle_present, status, cycle_indices = cycle_check_with_output(c, np.asarray(A_eq, dtype='float'), x0, basis) if status != 0: A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows(normalize_columns(R))) res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='revised simplex', options={'tol': 1e-12}, x0=x0) if res.status == 4: mp_print("Numerical difficulties with revised simplex, trying interior point method instead") res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='interior-point', options={'tol': 1e-12}) cycle_present = True if np.max(res.x) > 90 else False if cycle_present: cycle_indices = np.where(res.x > 90)[0] if np.any(np.isnan(res.x)): raise Exception('Remove cycles did not work, because LP-solver had issues. Try to solve this.') # if the objective is unbounded, there is a cycle that sums to zero while cycle_present: # Find minimal cycle met = -1 counter = 0 removable_is_ext = False while met < 0 and (not removable_is_ext): cycle_ind = cycle_indices[counter] met = get_remove_metabolite(R, network, cycle_ind) if met == -1: # Only external metabolites used in this ray, delete external and remember the ray removable_is_ext = True met = get_remove_metabolite(R, network, cycle_ind, allow_external=True) counter = counter + 1 if counter > len(cycle_indices): mp_print('No internal metabolite was found that was part of the cycle. This might cause problems.') if verbose: mp_print("Found a cycle, cancelling metabolite %d (%s) with reaction %d" % ( met, network.metabolites[met].id, cycle_ind)) R, external_cycle = cancel_with_cycle(R, met, cycle_ind, network, removable_is_ext=removable_is_ext) if external_cycle: external_cycles.append(external_cycle) count_since_last_redund = count_since_last_redund + 1 if count_since_last_redund > 10: count_since_last_redund = 0 R = np.transpose(R) rows_before = R.shape[0] if verbose: mp_print("\tDimensions before redund: %d %d" % (R.shape[0], R.shape[1])) start = time() R = redund(R) end = time() if verbose: mp_print("\tDimensions after redund: %d %d" % (R.shape[0], R.shape[1])) mp_print("\t\tRows removed by redund: %d" % (rows_before - R.shape[0])) mp_print("\tRedund took %f seconds" % (end - start)) R = np.transpose(R) A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr(normalize_columns(R)), only_eq=True) # Do new LP to check if there is still a cycle present. basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float')) b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10) cycle_present, status, cycle_indices = cycle_check_with_output(c, np.asarray(A_eq, dtype='float'), x0, basis) if status != 0: A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows(normalize_columns(R))) res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='revised simplex', options={'tol': 1e-12}, x0=x0) if res.status == 4: mp_print("Numerical difficulties with revised simplex, trying interior point method instead") res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='interior-point', options={'tol': 1e-12}) cycle_present = True if np.max(res.x) > 90 else False if cycle_present: cycle_indices = np.where(res.x > 90)[0] if np.any(np.isnan(res.x)): raise Exception('Remove cycles did not work, because LP-solver had issues. Try to solve this.') # Do redund one more time R = np.transpose(R) rows_before = R.shape[0] if verbose: mp_print("\tDimensions before redund: %d %d" % (R.shape[0], R.shape[1])) start = time() R = redund(R) end = time() if verbose: mp_print("\tDimensions after redund: %d %d" % (R.shape[0], R.shape[1])) mp_print("\t\tRows removed by redund: %d" % (rows_before - R.shape[0])) mp_print("\tRedund took %f seconds" % (end - start)) R = np.transpose(R) network.N = R if not len(external_cycles): external_cycles = None return R, network, external_cycles
def remove_cycles_redund(R, tol=1e-12, verbose=True): """Detect whether there are cycles, by doing an LP. If LP is unbounded find a minimal cycle. Cancel one metabolite with the cycle.""" zero_rays = np.where(np.count_nonzero(R, axis=0) == 0)[0] for ray_ind in zero_rays: R = np.delete(R, ray_ind, axis=1) cycle_rays = np.zeros((R.shape[0], 0)) A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr( normalize_columns(np.array(R, dtype='float'))), only_eq=True) if verbose: mp_print('Constructing basis for LP') basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float')) b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10) if verbose: mp_print('Starting linearity check using LP.') cycle_present, status, cycle_indices = cycle_check_with_output( c, np.asarray(A_eq, dtype='float'), x0, basis) if status != 0: print("Cycle check failed, trying normal LP") A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP( independent_rows_qr(normalize_columns(np.array(R, dtype='float')))) res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='revised simplex', options={'tol': 1e-12}, x0=x0) if res.status == 4: print( "Numerical difficulties with revised simplex, trying interior point method instead" ) res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='interior-point', options={'tol': 1e-12}) cycle_present = True if np.max(res.x) > 90 else False if cycle_present: cycle_indices = np.where(res.x > 90)[0] if np.any(np.isnan(res.x)): raise Exception( 'Remove cycles did not work, because LP-solver had issues. Try to solve this.' ) # if the objective is unbounded, there is a cycle that sums to zero while cycle_present: # Find minimal cycle met = -1 counter = 0 while met < 0: cycle_ind = cycle_indices[counter] met = get_remove_metabolite_redund(R, cycle_ind) counter = counter + 1 cycle_rays = np.append(cycle_rays, R[:, cycle_ind][:, np.newaxis], axis=1) R = cancel_with_cycle_redund(R, met, cycle_ind) # Do new LP to check if there is still a cycle present. A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr( normalize_columns(np.array(R, dtype='float'))), only_eq=True) basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float')) b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10) if verbose: mp_print('Starting linearity check in H_ineq using LP.') cycle_present, status, cycle_indices = cycle_check_with_output( c, np.asarray(A_eq, dtype='float'), x0, basis) if status != 0: print("Cycle check failed, trying normal LP") A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP( independent_rows_qr( normalize_columns(np.array(R, dtype='float')))) res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='revised simplex', options={'tol': 1e-12}, x0=x0) if res.status == 4: print( "Numerical difficulties with revised simplex, trying interior point method instead" ) res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='interior-point', options={'tol': 1e-12}) cycle_present = True if np.max(res.x) > 90 else False if cycle_present: cycle_indices = np.where(res.x > 90)[0] if np.any(np.isnan(res.x)): raise Exception( 'Remove cycles did not work, because LP-solver had issues. Try to solve this.' ) return R, cycle_rays