internal_ids = []
for metab in network.metabolites:
if not metab.is_external:
id_ind = [ind for ind, id in enumerate(ids) if id == metab.id]
if len(id_ind):
internal_ids.append(id_ind[0])
ids = list(np.delete(ids, internal_ids))
cone = np.delete(cone, internal_ids, axis=1)
if mpi_wrapper.is_first_process():
try:
np.savetxt(args.out_path,
cone,
delimiter=',',
header=','.join(ids),
comments='')
except OverflowError:
normalised = np.transpose(normalize_columns(np.transpose(cone)))
np.savetxt(args.out_path,
normalised,
delimiter=',',
header=','.join(ids),
comments='')
if args.print_conversions:
print_ecms_direct(np.transpose(cone), ids)
end = time()
mp_print('Ran in %f seconds' % (end - start))
def geometric_ray_adjacency(ray_matrix, plus=[-1], minus=[-1], tol=1e-3, verbose=True, remove_cycles=True):
"""
Returns r by r adjacency matrix of rays, given
ray matrix R. Diagonal is 0, not 1.
Calculated using LP adjacency test
:param R: ray matrix (columns are generating rays)
plus: indices of 'plus' columns
minus: indices of 'minus' columns
if plus/minus are not provided, find all adjacencies; otherwise only between each + and - pair
:return: r by r adjacency matrix
"""
start = time()
matrix_normalized = normalize_columns(ray_matrix)
matrix_indep_rows = independent_rows_qr(matrix_normalized)
# matrix_indep_rows = independent_rows(matrix_normalized)
# set default plus and minus
if len(plus) > 0 and plus[0] == -1:
plus = [x for x in range(matrix_indep_rows.shape[1])]
if len(minus) > 0 and minus[0] == -1:
minus = [x for x in range(matrix_indep_rows.shape[1])]
mpi_size = get_process_size()
mpi_rank = get_process_rank()
adjacency = []
nr_tests = len(plus) * len(minus)
# first find any column basis of R_indep
start_basis = get_basis_columns_qr(matrix_indep_rows)
start_basis_inv = np.linalg.inv(matrix_indep_rows[:, start_basis])
for i, p in enumerate(plus):
# add the plus ray into the basis
plus_basis = add_first_ray(matrix_indep_rows, start_basis_inv, start_basis, p)
plus_basis_inv = np.linalg.inv(matrix_indep_rows[:, plus_basis])
for j, m in enumerate(minus):
it = j + len(minus) * i
if it % mpi_size == mpi_rank:
# add the minus ray into the basis
plus_minus_basis = add_second_ray(matrix_indep_rows, plus_basis_inv, plus_basis, p, m)
res = determine_adjacency(matrix_indep_rows, p, m, plus_minus_basis)
if res == 1:
adjacency.append((p, m))
if it % (100 * mpi_size) == mpi_rank:
mp_print("Process %d is on adjacency test %d of %d (%f %%)" %
(mpi_rank, it, nr_tests, it / nr_tests * 100), PRINT_IF_RANK_NONZERO=True)
# bases = get_bases(matrix_indep_rows, plus, minus)
# for i in range(mpi_rank, nr_tests, mpi_size):
# plus_index = i // len(minus)
# minus_index = i % len(minus)
# basis = bases[plus_index, minus_index]
# res = determine_adjacency(matrix_indep_rows, plus[plus_index], minus[minus_index], basis)
# if res == 1:
# adjacency.append((plus[plus_index], minus[minus_index]))
# if i % 100 == 0:
# mp_print("Process %d is now on adjacency test %d" % (mpi_rank, i))
# MPI communication step
adj_sets = world_allgather(adjacency)
for i in range(mpi_size):
if i != mpi_rank:
adjacency.extend(adj_sets[i])
adjacency.sort()
end = time()
mp_print("Did LPs in %f seconds" % (end - start))
return adjacency
def drop_redundant_rays(ray_matrix,
verbose=True,
use_pre_filter=False,
rays_are_unique=True,
linearities=False,
normalised=True):
"""
:param ray_matrix:
:param verbose:
:param use_pre_filter: Sometimes, use_pre_filter=True can speed up the calculations, but mostly it doesn't
:param rays_are_unique: Boolean that states whether rays given as input are already unique
:param linearities: Boolean indicating if linearities are still present
:param normalised: Boolean indicating if ray_matrix columns are already normalised
:return:
"""
extreme_inds = []
non_extreme_inds = []
if not rays_are_unique:
# First make sure that no duplicate rays are in the matrix
ray_matrix = np.transpose(
unique(np.transpose(normalize_columns_fraction(ray_matrix))))
# Find 'cycles': combinations of columns of matrix_indep_rows that add up to zero, and remove them
if linearities:
if verbose:
mp_print('Detecting linearities in H_ineq.')
ray_matrix, cycle_rays = remove_cycles_redund(ray_matrix)
else:
cycle_rays = np.zeros((ray_matrix.shape[0], 0))
if not normalised:
if verbose:
mp_print('Normalizing columns.')
matrix_normalized = normalize_columns(ray_matrix)
else:
matrix_normalized = np.array(ray_matrix, dtype='float')
if verbose:
mp_print('Selecting independent rows.')
matrix_indep_rows = independent_rows_qr(matrix_normalized)
if use_pre_filter:
filtered_inds = pre_redund(matrix_indep_rows)
else:
filtered_inds = np.arange(matrix_indep_rows.shape[1])
mpi_size = mpi_wrapper.get_process_size()
mpi_rank = mpi_wrapper.get_process_rank()
matrix_indep_rows = matrix_indep_rows[:, filtered_inds]
# then find any column basis of R_indep
start_basis = get_basis_columns_qr(matrix_indep_rows)
start_basis_inv = np.linalg.inv(matrix_indep_rows[:, start_basis])
number_rays = matrix_indep_rows.shape[1]
for i in range(number_rays):
if i % mpi_size == mpi_rank:
if i % (10 * mpi_size) == mpi_rank:
mp_print(
"Process %d is on redundancy test %d of %d (%f %%). Found %d redundant rays."
% (mpi_rank, i, number_rays, i / number_rays * 100,
len(non_extreme_inds)),
PRINT_IF_RANK_NONZERO=True)
basis = add_first_ray(matrix_indep_rows, start_basis_inv,
start_basis, i)
extreme = check_extreme(matrix_indep_rows, i, basis)
if extreme:
extreme_inds.append(filtered_inds[i])
else:
non_extreme_inds.append(filtered_inds[i])
# MPI communication step
extreme_sets = mpi_wrapper.world_allgather(extreme_inds)
for i in range(mpi_size):
if i != mpi_rank:
extreme_inds.extend(extreme_sets[i])
extreme_inds.sort()
return extreme_inds, cycle_rays
def remove_cycles(R, network, tol=1e-12, verbose=True):
"""Detect whether there are cycles, by doing an LP. If LP is unbounded find a minimal cycle. Cancel one metabolite
with the cycle."""
external_cycles = []
count_since_last_redund = 0
A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr(normalize_columns(R)), only_eq=True)
# A_eqtest, b_eqtest, ctest, x0test = setup_cycle_LP(independent_rows(normalize_columns(R)), only_eq=True)
basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float'))
b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10)
cycle_present, status, cycle_indices = cycle_check_with_output(c, np.asarray(A_eq, dtype='float'), x0, basis)
if status != 0:
A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows(normalize_columns(R)))
res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='revised simplex', options={'tol': 1e-12},
x0=x0)
if res.status == 4:
mp_print("Numerical difficulties with revised simplex, trying interior point method instead")
res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='interior-point', options={'tol': 1e-12})
cycle_present = True if np.max(res.x) > 90 else False
if cycle_present:
cycle_indices = np.where(res.x > 90)[0]
if np.any(np.isnan(res.x)):
raise Exception('Remove cycles did not work, because LP-solver had issues. Try to solve this.')
# if the objective is unbounded, there is a cycle that sums to zero
while cycle_present:
# Find minimal cycle
met = -1
counter = 0
removable_is_ext = False
while met < 0 and (not removable_is_ext):
cycle_ind = cycle_indices[counter]
met = get_remove_metabolite(R, network, cycle_ind)
if met == -1: # Only external metabolites used in this ray, delete external and remember the ray
removable_is_ext = True
met = get_remove_metabolite(R, network, cycle_ind, allow_external=True)
counter = counter + 1
if counter > len(cycle_indices):
mp_print('No internal metabolite was found that was part of the cycle. This might cause problems.')
if verbose:
mp_print("Found a cycle, cancelling metabolite %d (%s) with reaction %d" % (
met, network.metabolites[met].id, cycle_ind))
R, external_cycle = cancel_with_cycle(R, met, cycle_ind, network, removable_is_ext=removable_is_ext)
if external_cycle:
external_cycles.append(external_cycle)
count_since_last_redund = count_since_last_redund + 1
if count_since_last_redund > 10:
count_since_last_redund = 0
R = np.transpose(R)
rows_before = R.shape[0]
if verbose:
mp_print("\tDimensions before redund: %d %d" % (R.shape[0], R.shape[1]))
start = time()
R = redund(R)
end = time()
if verbose:
mp_print("\tDimensions after redund: %d %d" % (R.shape[0], R.shape[1]))
mp_print("\t\tRows removed by redund: %d" % (rows_before - R.shape[0]))
mp_print("\tRedund took %f seconds" % (end - start))
R = np.transpose(R)
A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr(normalize_columns(R)), only_eq=True)
# Do new LP to check if there is still a cycle present.
basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float'))
b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10)
cycle_present, status, cycle_indices = cycle_check_with_output(c, np.asarray(A_eq, dtype='float'), x0, basis)
if status != 0:
A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows(normalize_columns(R)))
res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='revised simplex', options={'tol': 1e-12},
x0=x0)
if res.status == 4:
mp_print("Numerical difficulties with revised simplex, trying interior point method instead")
res = linprog(c, A_ub, b_ub, A_eq, b_eq, method='interior-point', options={'tol': 1e-12})
cycle_present = True if np.max(res.x) > 90 else False
if cycle_present:
cycle_indices = np.where(res.x > 90)[0]
if np.any(np.isnan(res.x)):
raise Exception('Remove cycles did not work, because LP-solver had issues. Try to solve this.')
# Do redund one more time
R = np.transpose(R)
rows_before = R.shape[0]
if verbose:
mp_print("\tDimensions before redund: %d %d" % (R.shape[0], R.shape[1]))
start = time()
R = redund(R)
end = time()
if verbose:
mp_print("\tDimensions after redund: %d %d" % (R.shape[0], R.shape[1]))
mp_print("\t\tRows removed by redund: %d" % (rows_before - R.shape[0]))
mp_print("\tRedund took %f seconds" % (end - start))
R = np.transpose(R)
network.N = R
if not len(external_cycles):
external_cycles = None
return R, network, external_cycles
def remove_cycles_redund(R, tol=1e-12, verbose=True):
"""Detect whether there are cycles, by doing an LP. If LP is unbounded find a minimal cycle. Cancel one metabolite
with the cycle."""
zero_rays = np.where(np.count_nonzero(R, axis=0) == 0)[0]
for ray_ind in zero_rays:
R = np.delete(R, ray_ind, axis=1)
cycle_rays = np.zeros((R.shape[0], 0))
A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr(
normalize_columns(np.array(R, dtype='float'))),
only_eq=True)
if verbose:
mp_print('Constructing basis for LP')
basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float'))
b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10)
if verbose:
mp_print('Starting linearity check using LP.')
cycle_present, status, cycle_indices = cycle_check_with_output(
c, np.asarray(A_eq, dtype='float'), x0, basis)
if status != 0:
print("Cycle check failed, trying normal LP")
A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP(
independent_rows_qr(normalize_columns(np.array(R, dtype='float'))))
res = linprog(c,
A_ub,
b_ub,
A_eq,
b_eq,
method='revised simplex',
options={'tol': 1e-12},
x0=x0)
if res.status == 4:
print(
"Numerical difficulties with revised simplex, trying interior point method instead"
)
res = linprog(c,
A_ub,
b_ub,
A_eq,
b_eq,
method='interior-point',
options={'tol': 1e-12})
cycle_present = True if np.max(res.x) > 90 else False
if cycle_present:
cycle_indices = np.where(res.x > 90)[0]
if np.any(np.isnan(res.x)):
raise Exception(
'Remove cycles did not work, because LP-solver had issues. Try to solve this.'
)
# if the objective is unbounded, there is a cycle that sums to zero
while cycle_present:
# Find minimal cycle
met = -1
counter = 0
while met < 0:
cycle_ind = cycle_indices[counter]
met = get_remove_metabolite_redund(R, cycle_ind)
counter = counter + 1
cycle_rays = np.append(cycle_rays,
R[:, cycle_ind][:, np.newaxis],
axis=1)
R = cancel_with_cycle_redund(R, met, cycle_ind)
# Do new LP to check if there is still a cycle present.
A_eq, b_eq, c, x0 = setup_cycle_LP(independent_rows_qr(
normalize_columns(np.array(R, dtype='float'))),
only_eq=True)
basis = get_basis_columns_qr(np.asarray(A_eq, dtype='float'))
b_eq, x0 = perturb_LP(b_eq, x0, A_eq, basis, 1e-10)
if verbose:
mp_print('Starting linearity check in H_ineq using LP.')
cycle_present, status, cycle_indices = cycle_check_with_output(
c, np.asarray(A_eq, dtype='float'), x0, basis)
if status != 0:
print("Cycle check failed, trying normal LP")
A_ub, b_ub, A_eq, b_eq, c, x0 = setup_cycle_LP(
independent_rows_qr(
normalize_columns(np.array(R, dtype='float'))))
res = linprog(c,
A_ub,
b_ub,
A_eq,
b_eq,
method='revised simplex',
options={'tol': 1e-12},
x0=x0)
if res.status == 4:
print(
"Numerical difficulties with revised simplex, trying interior point method instead"
)
res = linprog(c,
A_ub,
b_ub,
A_eq,
b_eq,
method='interior-point',
options={'tol': 1e-12})
cycle_present = True if np.max(res.x) > 90 else False
if cycle_present:
cycle_indices = np.where(res.x > 90)[0]
if np.any(np.isnan(res.x)):
raise Exception(
'Remove cycles did not work, because LP-solver had issues. Try to solve this.'
)
return R, cycle_rays