def eval_f(Y):
""" some reformulations to make eval_f_orig
compatible with algopy
missing: support for scipy.linalg.expm
i.e., this function can't be differentiated with algopy
"""
a, b, v = transform_params(Y)
Q = algopy.zeros((4,4), dtype=Y)
Q[0,0] = 0; Q[0,1] = a; Q[0,2] = b; Q[0,3] = b;
Q[1,0] = a; Q[1,1] = 0; Q[1,2] = b; Q[1,3] = b;
Q[2,0] = b; Q[2,1] = b; Q[2,2] = 0; Q[2,3] = a;
Q[3,0] = b; Q[3,1] = b; Q[3,2] = a; Q[3,3] = 0;
Q = Q * v
Q -= algopy.diag(algopy.sum(Q, axis=1))
#P = linalg.expm(Q)
# XXX can I get rid of the 4 on the following line?
P = algopy_expm(Q, 4)
S = algopy.log(algopy.dot(algopy.diag(v), P))
return -algopy.sum(S * g_data)
def eval_f(Y):
""" some reformulations to make eval_f_orig
compatible with algopy
missing: support for scipy.linalg.expm
i.e., this function can't be differentiated with algopy
"""
a, b, v = transform_params(Y)
Q = algopy.zeros((4, 4), dtype=Y)
Q[0, 0] = 0
Q[0, 1] = a
Q[0, 2] = b
Q[0, 3] = b
Q[1, 0] = a
Q[1, 1] = 0
Q[1, 2] = b
Q[1, 3] = b
Q[2, 0] = b
Q[2, 1] = b
Q[2, 2] = 0
Q[2, 3] = a
Q[3, 0] = b
Q[3, 1] = b
Q[3, 2] = a
Q[3, 3] = 0
Q = Q * v
Q -= algopy.diag(algopy.sum(Q, axis=1))
P = algopy.expm(Q)
S = algopy.log(algopy.dot(algopy.diag(v), P))
return -algopy.sum(S * g_data)
def pre_Q_to_Q(pre_Q, stationary_distn, target_expected_rate):
"""
Return a matrix with a different diagonal and a different scaling.
"""
unscaled_Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
r = -algopy.dot(algopy.diag(unscaled_Q), stationary_distn)
Q = (target_expected_rate / r) * unscaled_Q
return Q
def get_Q_unconstrained_kb(
ts, tv, syn, nonsyn, compo, asym_compo,
h,
log_counts,
log_mu, log_kappa, log_omega, d, log_kb, log_nt_weights):
"""
This adds yet another parameter.
"""
#FIXME: constructing this each time seems wasteful
codon_neighbor_mask = ts + tv
#FIXME: this is being hacked to use fixed-order quadrature
#FIXME: and to disregard the h parameter
mu = algopy.exp(log_mu)
kappa = algopy.exp(log_kappa)
omega = algopy.exp(log_omega)
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
H = get_fixation_unconstrained_kb_fquad(
S, d, log_kb, g_quad_x, g_quad_w, codon_neighbor_mask)
#H = get_fixation_unconstrained_kb_fquad_cython(
#S, d, log_kb, codon_neighbor_mask)
pre_Q = mu * (kappa * ts + tv) * (omega * nonsyn + syn) * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * H
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def get_Q(
ts, tv, syn, nonsyn, compo, asym_compo,
h,
log_counts,
log_mu, log_kappa, log_omega, log_nt_weights):
"""
Notation is from Yang and Nielsen 2008.
The first group of args consists of precomputed ndarrays.
The second group is only the fixation function.
The third group consists of empirically (non-free) estimated parameters.
The fourth group depends only on free parameters.
@param ts: indicator for transition
@param tv: indicator for transversion
@param syn: indicator for synonymous codons
@param nonsyn: indicator for nonsynonymous codons
@param compo: site independent nucleotide composition per codon
@param asym_compo: tensor from get_asym_compo function
@param h: fixation function
@param log_counts: empirically counted codons in the data set
@param log_mu: free param for scaling
@param log_kappa: free param for transition transversion rate distinction
@param log_omega: free param for syn nonsyn rate distinction
@param log_nt_weights: mostly free param array for mutation equilibrium
@return: rate matrix
"""
mu = algopy.exp(log_mu)
kappa = algopy.exp(log_kappa)
omega = algopy.exp(log_omega)
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
pre_Q = mu * (kappa * ts + tv) * (omega * nonsyn + syn) * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * h(S)
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def centered_tree_covariance(B, nleaves, v):
"""
@param B: rows of this unweighted incidence matrix are edges
@param nleaves: number of leaves
@param v: vector of edge variances
"""
#TODO: track the block multiplication through the schur complement
W = diag(reciprocal(v))
L = dot(B.T, dot(W, B))
#print('full laplacian matrix:')
#print(L)
#print()
nvertices = v.shape[0]
ninternal = nvertices - nleaves
Laa = L[:nleaves, :nleaves]
Lab = L[:nleaves, nleaves:]
Lba = L[nleaves:, :nleaves]
Lbb = L[nleaves:, nleaves:]
L_schur = Laa - dot(Lab, dot(inv(Lbb), Lba))
L_schur_pinv = restored(inv(augmented(L_schur)))
#print('schur laplacian matrix:')
#print(L_schur)
#print()
#print('pinv of schur laplacian matrix:')
#print(L_schur_pinv)
#print()
return L_schur_pinv
def get_Q_slsqp(
ts, tv, syn, nonsyn, compo, asym_compo,
h,
log_counts, v,
theta):
#FIXME: hardcoded for selection without recessivity parameters
#
# unpack theta
branch_length = theta[0]
kappa = theta[1]
omega = theta[2]
"""
nt_probs = algopy.zeros(4, dtype=theta)
nt_probs[0] = theta[3]
nt_probs[1] = theta[4]
nt_probs[2] = theta[5]
nt_probs[3] = 1.0 - algopy.sum(nt_probs)
print nt_probs
log_nt_weights = algopy.log(nt_probs)
"""
log_nt_weights = theta[-4:]
#
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
pre_Q_exch = (kappa * ts + tv) * (omega * nonsyn + syn)
pre_Q = pre_Q_exch * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * h(S)
rates = algopy.sum(pre_Q, axis=1)
Q = pre_Q - algopy.diag(rates)
Q *= branch_length / algopy.dot(rates, v)
return Q
def get_Q(
ts, tv, syn, nonsyn, compo, asym_compo,
h,
log_counts,
log_mu, log_kappa, log_omega, log_nt_weights):
"""
Notation is from Yang and Nielsen 2008.
The first group of args consists of precomputed ndarrays.
The second group is only the fixation function.
The third group consists of empirically (non-free) estimated parameters.
The fourth group depends only on free parameters.
Speed matters.
@param ts: indicator for transition
@param tv: indicator for transversion
@param syn: indicator for synonymous codons
@param nonsyn: indicator for nonsynonymous codons
@param compo: site independent nucleotide composition per codon
@param asym_compo: tensor from get_asym_compo function
@param h: fixation function
@param log_counts: empirically counted codons in the data set
@param log_mu: free param for scaling
@param log_kappa: free param for transition transversion rate distinction
@param log_omega: free param for syn nonsyn rate distinction
@param log_nt_weights: mostly free param array for mutation equilibrium
@return: rate matrix
"""
mu = algopy.exp(log_mu)
kappa = algopy.exp(log_kappa)
omega = algopy.exp(log_omega)
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
pre_Q = mu * (kappa * ts + tv) * (omega * nonsyn + syn) * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * h(S)
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def get_log_likelihood(pre_Q_prefix, pre_Q_suffix, v, subs_counts):
"""
The stationary distribution of P is empirically derived.
It is proportional to the codon counts by construction.
@param pre_Q_prefix: component of hadamard decomposition of pre_Q
@param pre_Q_suffix: component of hadamard decomposition of pre_Q
@param v: stationary distribution proportional to observed codon counts
@param subs_counts: observed substitution counts
"""
Q = get_Q(pre_Q_prefix, pre_Q_suffix)
#
P = algopy.expm(Q)
#
# This untested eigh approach is way too slow because of the algopy eigh.
"""
Da = numpy.diag(numpy.sqrt(v))
Db = numpy.diag(numpy.reciprocal(numpy.sqrt(v)))
Q_symmetrized = algopy.dot(Da, algopy.dot(Q, Db))
w, V = algopy.eigh(Q_symmetrized)
W_exp = algopy.diag(algopy.exp(w))
P_symmetrized = algopy.dot(V, algopy.dot(W_exp, V.T))
P = algopy.dot(Db, algopy.dot(P_symmetrized, Da))
"""
#
log_score_matrix = algopy.log(algopy.dot(algopy.diag(v), P))
log_likelihood = algopy.sum(log_score_matrix * subs_counts)
return log_likelihood
def get_Q(
gtr, syn, nonsyn, compo, asym_compo,
h,
log_counts,
log_mu, log_g, log_omega, log_nt_weights):
"""
Most of the notation is from Yang and Nielsen 2008.
The first group of args consists of precomputed ndarrays.
The second group is only the fixation function.
The third group consists of empirically (non-free) estimated parameters.
The fourth group depends only on free parameters.
@param gtr: ndim-3 ndarray indicating the nucleotide exchange type
@param syn: indicator for synonymous codons
@param nonsyn: indicator for nonsynonymous codons
@param compo: site independent nucleotide composition per codon
@param asym_compo: tensor from get_asym_compo function
@param h: fixation function
@param log_counts: empirically counted codons in the data set
@param log_mu: free param for scaling
@param log_g: logs of six exchangeabilities
@param log_omega: free param for syn nonsyn rate distinction
@param log_nt_weights: mostly free param array for mutation equilibrium
@return: rate matrix
"""
mu = algopy.exp(log_mu)
g = algopy.exp(log_g)
omega = algopy.exp(log_omega)
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
pre_Q = mu * algopy.dot(gtr, g) * (omega * nonsyn + syn) * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * h(S)
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def create_transition_matrix_numeric(mu, d, v):
"""
Use numerical integration.
This is not so compatible with algopy because it goes through fortran.
Note that d = 2*h - 1 following Kimura 1957.
The rate mu is a catch-all scaling factor.
The finite distribution v is assumed to be a stochastic vector.
@param mu: scales the rate matrix
@param d: dominance (as opposed to recessiveness) of preferred states.
@param v: numpy array defining a distribution over states
@return: transition matrix
"""
# Construct the numpy matrix whose entries
# are differences of log equilibrium probabilities.
# Everything in this code block is pure numpy.
F = numpy.log(v)
e = numpy.ones_like(F)
S = numpy.outer(e, F) - numpy.outer(F, e)
# Create the rate matrix Q and return its matrix exponential.
# Things in this code block may use algopy if mu and d
# are bundled with truncated Taylor information.
D = d * numpy.sign(S)
pre_Q = numpy.vectorize(numeric_fixation)(0.5*S, D)
pre_Q = mu * pre_Q
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
P = algopy.expm(Q)
return P
def create_transition_matrix_numeric(mu, d, v):
"""
Use numerical integration.
This is not so compatible with algopy because it goes through fortran.
Note that d = 2*h - 1 following Kimura 1957.
The rate mu is a catch-all scaling factor.
The finite distribution v is assumed to be a stochastic vector.
@param mu: scales the rate matrix
@param d: dominance (as opposed to recessiveness) of preferred states.
@param v: numpy array defining a distribution over states
@return: transition matrix
"""
# Construct the numpy matrix whose entries
# are differences of log equilibrium probabilities.
# Everything in this code block is pure numpy.
F = numpy.log(v)
e = numpy.ones_like(F)
S = numpy.outer(e, F) - numpy.outer(F, e)
# Create the rate matrix Q and return its matrix exponential.
# Things in this code block may use algopy if mu and d
# are bundled with truncated Taylor information.
D = d * numpy.sign(S)
pre_Q = numpy.vectorize(numeric_fixation)(0.5 * S, D)
pre_Q = mu * pre_Q
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
P = algopy.expm(Q)
return P
def clever_cross_entropy_trees(B, nleaves, va, vb):
"""
Try being a little more clever.
@param B: augmented incidence matrix
@param nleaves: number of leaves
@param va: augmented reference point edge variances
@param vb: augmented test point edge variances
"""
# deduce some quantities assuming an unrooted bifurcating tree
ninternal = nleaves - 2
nvertices = nleaves + ninternal
nedges = nvertices - 1
# define an index for taking schur complements
n = nvertices
k = nleaves + 1
# Construct the full Laplacian matrix plus J/n.
# Take a block of the diagonal, corresponding to the inverse
# of a schur complement.
Wa = diag(reciprocal(va))
La_plus = dot(B.T, dot(Wa, B))
print(La_plus)
print(scipy.linalg.eigh(La_plus))
Laa = La_plus[:k, :k]
Lab = La_plus[:k, k:]
Lba = La_plus[k:, :k]
Lbb = La_plus[k:, k:]
L_schur_plus = Laa - dot(Lab, dot(inv(Lbb), Lba))
assert_allclose(inv(L_schur_plus), inv(La_plus)[:k, :k])
A = inv(La_plus)[:k, :k]
print(scipy.linalg.eigh(A))
# Construct the Schur complement of the test point matrix.
Wb = diag(reciprocal(vb))
L_plus = dot(B.T, dot(Wb, B))
Laa = L_plus[:k, :k]
Lab = L_plus[:k, k:]
Lba = L_plus[k:, :k]
Lbb = L_plus[k:, k:]
L_schur_plus = Laa - dot(Lab, dot(inv(Lbb), Lba))
B_inv = L_schur_plus
#return 0.5 * ((n-1) * LOG2PI + trace(dot(B_inv, A)) - log(det(B_inv)))
return 0.5 * (n * LOG2PI + trace(dot(B_inv, A) - 1) - log(det(B_inv)))
def get_Q(pre_Q_prefix, pre_Q_suffix):
"""
@param pre_Q_prefix: this is an algopy aware ndarray
@param pre_Q_suffix: this is a numpy ndarray
@return: an algopy aware ndarray
"""
pre_Q = pre_Q_prefix * pre_Q_suffix
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def guess_branch_length(subs_counts):
"""
Make a very crude guess of expected number of changes along a branch.
@param subs_counts: an (nstates, nstates) ndarray of observed substitutions
@return: crude guess of expected number of changes along the branch
"""
total_count = algopy.sum(subs_counts)
diag_count = algopy.sum(algopy.diag(subs_counts))
crude_estimate = (total_count - diag_count) / float(total_count)
return crude_estimate
def get_Q_unconstrained(gtr, syn, nonsyn, compo, asym_compo, h, log_counts,
log_mu, log_g, log_omega, d, log_nt_weights):
mu = algopy.exp(log_mu)
g = algopy.exp(log_g)
omega = algopy.exp(log_omega)
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
pre_Q = mu * algopy.dot(gtr, g) * (omega * nonsyn + syn) * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * h(S, d)
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def get_branch_mix(probs, pre_Qs, eq_distns, branch_length):
"""
This log likelihood calculation function is compatible with algopy.
Note that the word 'mix' in the function name
does not refer to a mix of branch lengths,
but rather to a mixture of unscaled parameterized rate matrices.
@param probs: discrete distribution of mixture probabilities
@param pre_Qs: rates with arbitrary scaling and arbitrary diagonals
@param eq_distns: equilibrium distributions
@param branch_length: expected number of changes
@return: transition matrices
"""
# Subtract diagonals to give the unscaled rate matrices.
# Also compute the expected rates of the unscaled rate matrices.
# Use an unnecessarily explicit-looking calculation,
# because the entries inside the probs list
# and the entries inside the observed expected rates list
# each have taylor information,
# but the lists themselves are not taylor-aware.
# The code could be re-orgainized later so that we are using
# more explicitly taylor-aware lists.
unscaled_Qs = []
r = 0
for p, pre_Q, eq_distn in zip(probs, pre_Qs, eq_distns):
unscaled_Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
unscaled_Qs.append(unscaled_Q)
observed_r = -algopy.dot(algopy.diag(unscaled_Q), eq_distn)
r = r + p * observed_r
# Compute the correctly scaled rate matrices
# so that the expected rate of the mixture is equal
# to the branch length that has been passed as an argument
# to this function.
Qs = []
for unscaled_Q in unscaled_Qs:
Q = (branch_length / r) * unscaled_Q
Qs.append(Q)
# Return the appropriately time-scaled transition matrices.
return [algopy.expm(Q) for Q in Qs]
def eval_f_eigh(Y):
""" some reformulations to make eval_f_orig
compatible with algopy
replaced scipy.linalg.expm by a symmetric eigenvalue decomposition
this function **can** be differentiated with algopy
"""
a, b, v = transform_params(Y)
Q = algopy.zeros((4,4), dtype=Y)
Q[0,0] = 0; Q[0,1] = a; Q[0,2] = b; Q[0,3] = b;
Q[1,0] = a; Q[1,1] = 0; Q[1,2] = b; Q[1,3] = b;
Q[2,0] = b; Q[2,1] = b; Q[2,2] = 0; Q[2,3] = a;
Q[3,0] = b; Q[3,1] = b; Q[3,2] = a; Q[3,3] = 0;
Q = algopy.dot(Q, algopy.diag(v))
Q -= algopy.diag(algopy.sum(Q, axis=1))
va = algopy.diag(algopy.sqrt(v))
vb = algopy.diag(1./algopy.sqrt(v))
W, U = algopy.eigh(algopy.dot(algopy.dot(va, Q), vb))
M = algopy.dot(U, algopy.dot(algopy.diag(algopy.exp(W)), U.T))
P = algopy.dot(vb, algopy.dot(M, va))
S = algopy.log(algopy.dot(algopy.diag(v), P))
return -algopy.sum(S * g_data)
def test_svd(self):
D, P, M, N = 3, 1, 5, 2
A = UTPM(numpy.random.random((D, P, M, N)))
U, s, V = svd(A)
S = zeros((M, N), dtype=A)
S[:N, :N] = diag(s)
assert_array_almost_equal((dot(dot(U, S), V.T) - A).data, 0.)
assert_array_almost_equal((dot(U.T, U) - numpy.eye(M)).data, 0.)
assert_array_almost_equal((dot(U, U.T) - numpy.eye(M)).data, 0.)
assert_array_almost_equal((dot(V.T, V) - numpy.eye(N)).data, 0.)
assert_array_almost_equal((dot(V, V.T) - numpy.eye(N)).data, 0.)
def get_Q_unconstrained_kb(
gtr, syn, nonsyn, compo, asym_compo,
h,
log_counts,
log_mu, log_g, log_omega, d, log_kb, log_nt_weights):
mu = algopy.exp(log_mu)
g = algopy.exp(log_g)
omega = algopy.exp(log_omega)
F = get_selection_F(log_counts, compo, log_nt_weights)
S = get_selection_S(F)
pre_Q = mu * algopy.dot(gtr, g) * (omega * nonsyn + syn) * algopy.exp(
algopy.dot(asym_compo, log_nt_weights)) * h(S, d, log_kb)
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
return Q
def test_svd(self):
D,P,M,N = 3,1,5,2
A = UTPM(numpy.random.random((D,P,M,N)))
U,s,V = svd(A)
S = zeros((M,N),dtype=A)
S[:N,:N] = diag(s)
assert_array_almost_equal( (dot(dot(U, S), V.T) - A).data, 0.)
assert_array_almost_equal( (dot(U.T, U) - numpy.eye(M)).data, 0.)
assert_array_almost_equal( (dot(U, U.T) - numpy.eye(M)).data, 0.)
assert_array_almost_equal( (dot(V.T, V) - numpy.eye(N)).data, 0.)
assert_array_almost_equal( (dot(V, V.T) - numpy.eye(N)).data, 0.)
def eval_f_explicit(subs_counts, v, Y):
"""
Note that Y is last for compatibility with functools.partial.
It is convenient for usage with numdifftools, although this parameter
ordering is the opposite of the convention of scipy.optimize.
@return: negative log likelihood
@param Y: parameters to jointly estimate
@param subs_counts: observed data
@param v: fixed equilibrium probabilities for states
"""
P = create_transition_matrix_explicit(Y, v)
vdiag = algopy.diag(v)
J = algopy.dot(vdiag, P)
S = algopy.log(J)
return -algopy.sum(S * subs_counts)
def eval_f_explicit(subs_counts, v, Y):
"""
Note that Y is last for compatibility with functools.partial.
It is convenient for usage with numdifftools, although this parameter
ordering is the opposite of the convention of scipy.optimize.
@return: negative log likelihood
@param Y: parameters to jointly estimate
@param subs_counts: observed data
@param v: fixed equilibrium probabilities for states
"""
P = create_transition_matrix_explicit(Y, v)
vdiag = algopy.diag(v)
J = algopy.dot(vdiag, P)
S = algopy.log(J)
return -algopy.sum(S * subs_counts)
def create_transition_matrix_explicit(Y, v):
"""
Use hypergeometric functions.
Note that d = 2*h - 1 following Kimura 1957.
The rate mu is a catch-all scaling factor.
The finite distribution v is assumed to be a stochastic vector.
@param Y: vector of parameters to optimize
@param v: numpy array defining a distribution over states
@return: transition matrix
"""
n = len(v)
mu, d = transform_params(Y)
# Construct the numpy matrix whose entries
# are differences of log equilibrium probabilities.
# Everything in this code block is pure numpy.
F = numpy.log(v)
e = numpy.ones_like(F)
S = numpy.outer(e, F) - numpy.outer(F, e)
# Create the rate matrix Q and return its matrix exponential.
# Things in this code block may use algopy if mu and d
# are bundled with truncated Taylor information.
D = d * numpy.sign(S)
#FIXME: I would like to further vectorize this block,
# and also it may currently give subtly wrong results
# because denom_piecewise may not vectorize correctly.
pre_Q = algopy.zeros((n, n), dtype=Y)
for i in range(n):
for j in range(n):
pre_Q[i, j] = 1. / denom_piecewise(0.5 * S[i, j], D[i, j])
pre_Q = mu * pre_Q
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
P = algopy.expm(Q)
return P
def create_transition_matrix_explicit(Y, v):
"""
Use hypergeometric functions.
Note that d = 2*h - 1 following Kimura 1957.
The rate mu is a catch-all scaling factor.
The finite distribution v is assumed to be a stochastic vector.
@param Y: vector of parameters to optimize
@param v: numpy array defining a distribution over states
@return: transition matrix
"""
n = len(v)
mu, d = transform_params(Y)
# Construct the numpy matrix whose entries
# are differences of log equilibrium probabilities.
# Everything in this code block is pure numpy.
F = numpy.log(v)
e = numpy.ones_like(F)
S = numpy.outer(e, F) - numpy.outer(F, e)
# Create the rate matrix Q and return its matrix exponential.
# Things in this code block may use algopy if mu and d
# are bundled with truncated Taylor information.
D = d * numpy.sign(S)
#FIXME: I would like to further vectorize this block,
# and also it may currently give subtly wrong results
# because denom_piecewise may not vectorize correctly.
pre_Q = algopy.zeros((n,n), dtype=Y)
for i in range(n):
for j in range(n):
pre_Q[i, j] = 1. / denom_piecewise(0.5*S[i, j], D[i, j])
pre_Q = mu * pre_Q
Q = pre_Q - algopy.diag(algopy.sum(pre_Q, axis=1))
P = algopy.expm(Q)
return P
def eval_f_eigh(Y):
""" some reformulations to make eval_f_orig
compatible with algopy
replaced scipy.linalg.expm by a symmetric eigenvalue decomposition
this function **can** be differentiated with algopy
"""
a, b, v = transform_params(Y)
Q = algopy.zeros((4, 4), dtype=Y)
Q[0, 0] = 0
Q[0, 1] = a
Q[0, 2] = b
Q[0, 3] = b
Q[1, 0] = a
Q[1, 1] = 0
Q[1, 2] = b
Q[1, 3] = b
Q[2, 0] = b
Q[2, 1] = b
Q[2, 2] = 0
Q[2, 3] = a
Q[3, 0] = b
Q[3, 1] = b
Q[3, 2] = a
Q[3, 3] = 0
Q = algopy.dot(Q, algopy.diag(v))
Q -= algopy.diag(algopy.sum(Q, axis=1))
va = algopy.diag(algopy.sqrt(v))
vb = algopy.diag(1. / algopy.sqrt(v))
W, U = algopy.eigh(algopy.dot(algopy.dot(va, Q), vb))
M = algopy.dot(U, algopy.dot(algopy.diag(algopy.exp(W)), U.T))
P = algopy.dot(vb, algopy.dot(M, va))
S = algopy.log(algopy.dot(algopy.diag(v), P))
return -algopy.sum(S * g_data)
def submain_constrained_dominance(args):
#
# Precompute some ndarrays
# according to properties of DNA and the genetic code.
if args.mtdna or args.force_mtcode:
code = npcodon.g_code_mito
stop = npcodon.g_stop_mito
else:
code = npcodon.g_code
stop = npcodon.g_stop
#
all_codons = npcodon.enum_codons(stop)
codons = all_codons[:-len(stop)]
ts, tv = npcodon.get_ts_tv(codons)
syn, nonsyn = npcodon.get_syn_nonsyn(code, codons)
compo = npcodon.get_compo(codons)
asym_compo = npcodon.get_asym_compo(codons)
ham = npcodon.get_hamming(codons)
#
subs_counts = yangdata.get_subs_counts_from_data_files(args)
codon_counts = numpy.sum(subs_counts, axis=0) + numpy.sum(
subs_counts, axis=1)
for a, b in zip(codons, codon_counts):
print a, ':', b
print 'raw codon total:', numpy.sum(codon_counts)
print 'raw codon counts:', codon_counts
codon_counts = codon_counts[:len(codons)]
print 'non-stop codon total:', numpy.sum(codon_counts)
subs_counts = subs_counts[:len(codons), :len(codons)]
v = codon_counts / float(numpy.sum(codon_counts))
log_counts = numpy.log(codon_counts)
#
if args.disease == 'genic':
h = get_fixation_genic
elif args.disease == 'recessive':
h = get_fixation_recessive_disease
elif args.disease == 'dominant':
h = get_fixation_dominant_disease
else:
raise Exception
#
# initialize parameter values
mu_r = 1.0
kappa = 2.0
omega = 0.1
#pA = 0.25
#pC = 0.25
#pG = 0.25
#pT = 0.25
theta = numpy.array([mu_r, kappa, omega, 0, 0, 0, 0])
#
# adjust the expected rate parameter
Q = get_Q_slsqp(
ts, tv, syn, nonsyn, compo, asym_compo,
h,
log_counts, v,
theta)
expected_rate = -algopy.dot(algopy.diag(Q), v)
mu_n = 1. / expected_rate
mu_r = npcodon.get_lb_expected_subs(ham, subs_counts)
theta = numpy.array([mu_r, kappa, omega, 0, 0, 0, 0])
#
# get the log likelihood associated with the initial guess
fmin_args = (
subs_counts, log_counts, v,
h,
ts, tv, syn, nonsyn, compo, asym_compo,
)
initial_cost = eval_f(theta, *fmin_args)
print 'negative log likelihood of initial guess:',
print initial_cost
print
print 'entropy bound on negative log likelihood:',
print npcodon.get_lb_neg_ll(subs_counts)
print
do_opt(args, eval_f, theta, fmin_args)
def rhfenergy(alpha_old, coef2, xyz, l, charges, xyz_atom, natoms, nbasis,
contr_list, ne, max_scf, max_d, log, eigen, printguess,
readguess, name, write, dtype):
'''
This function returns the rhf function
Parameters:
alpha_old : array
Gaussian exponents
coef2 : array
Contraction coeffients
xyz : array 3N
Gaussian centers
l : array 3N
Angular momentum each entry is a vector
eg. s orbital (0,0,0) or pz (1,0,0)
charges : array
Atom charges
nbasis : int
Number of basis
contr_list: list of integers
Specify the number of orbitals in each atom
ne : int
Number of electrons
max_scf : int
maximum number of scf cycles
log : bool
The exponents are given in log
printguess: str or None
File to print coeff matrix initial guess
readguess : str or None
File that contains coeff matrix initial guess
name : str
Output file name
write : bool
True if printing
dtype : type of output
This is the directive to know if algopy will be used or not
np.float64(1.0) if it is a single point calculation
otherwise, it specify the size of the UTMP, autodifferentiation
Returns:
energy : float
RHF energy
'''
tool_D = 1e-8
tool = 1e-8
if log:
alpha = algopy.exp(alpha_old)
else:
alpha = alpha_old
if type(xyz_atom) != np.ndarray: ## Cover the case of diff xyz atom
coef = normalization(alpha,
coef2,
l,
contr_list,
dtype=np.float64(1.0))
V = nuclearmatrix(alpha,
coef,
xyz,
l,
nbasis,
charges,
xyz_atom,
natoms,
contr_list,
dtype=dtype)
S = overlapmatrix(alpha,
coef,
xyz,
l,
nbasis,
contr_list,
dtype=np.float64(1.0))
T = kineticmatrix(alpha,
coef,
xyz,
l,
nbasis,
contr_list,
dtype=np.float64(1.0))
Eri = erivector(alpha,
coef,
xyz,
l,
nbasis,
contr_list,
dtype=np.float(1.0))
else:
coef = normalization(alpha, coef2, l, contr_list, dtype=dtype)
S = overlapmatrix(alpha, coef, xyz, l, nbasis, contr_list, dtype=dtype)
V = nuclearmatrix(alpha,
coef,
xyz,
l,
nbasis,
charges,
xyz_atom,
natoms,
contr_list,
dtype=dtype)
T = kineticmatrix(alpha, coef, xyz, l, nbasis, contr_list, dtype=dtype)
Eri = erivector(alpha, coef, xyz, l, nbasis, contr_list, dtype=dtype)
Hcore = T + V
if eigen:
eigsys = eigensolver(S)
SqrtLambda = algopy.diag(1. / algopy.sqrt(eigsys[0]))
L = eigsys[1]
LT = algopy.transpose(L)
SqrtS = algopy.dot(algopy.dot(L, SqrtLambda), LT)
SqrtST = algopy.transpose(SqrtS)
else:
Sinv = np.linalg.inv(S)
if readguess != None:
C = np.load(readguess)
D = np.zeros((nbasis, nbasis))
for i in range(nbasis):
for j in range(nbasis):
tmp = 0.0
for k in range(ne):
tmp = tmp + C[i, k] * C[j, k]
D[i, j] = tmp
F = fockmatrix(Hcore, Eri, D, nbasis, alpha, dtype)
else:
F = Hcore
OldE = 1e8
status = False
E_step = []
for scf_iter in range(max_scf):
if eigen:
Fprime = algopy.dot(algopy.dot(SqrtST, F), SqrtS)
eigsysFockOp = eigensolver(Fprime)
Cprime = eigsysFockOp[1]
C = algopy.dot(SqrtS, Cprime)
Fprime = algopy.dot(algopy.dot(SqrtST, F), SqrtS)
eigsysFockOp = eigensolver(Fprime)
Cprime = eigsysFockOp[1]
C = algopy.dot(SqrtS, Cprime)
D = algopy.zeros((nbasis, nbasis), dtype=dtype)
for i in range(nbasis):
for j in range(nbasis):
tmp = 0.0
for k in range(ne):
tmp = tmp + C[i, k] * C[j, k]
D[i, j] = tmp
else:
D = newdensity(F, Sinv, nbasis, ne)
for i in range(max_d):
D = cannonicalputication(D, S)
err = np.linalg.norm(D - np.dot(np.dot(D, S), D))
if err < tool_D:
break
F = fockmatrix(Hcore, Eri, D, nbasis, alpha, dtype)
E_elec = algopy.sum(np.multiply(D, Hcore + F))
E_step.append(E_elec)
E_nuc = nuclearrepulsion(xyz_atom, charges, natoms)
if np.absolute(E_elec - OldE) < tool:
status = True
break
OldE = E_elec
E_nuc = nuclearrepulsion(xyz_atom, charges, natoms)
if printguess != None:
np.save(printguess, C)
def update_system():
mol.energy = E_elec + E_nuc
mol.erepulsion = Eri
mol.hcore = Hcore
mol.mo_coeff = C
return
def write_molden():
import Data
from Data import select_atom
## Details of calculation
tape.write('[Energy] \n')
tape.write('E_elec: ' + str(E_elec) + '\n')
tape.write('E_nuc: ' + str(E_nuc) + '\n')
tape.write('E_tot: ' + str(E_nuc + E_elec) + '\n')
tape.write('SCF Details\n')
line = 'Eigen: '
if eigen:
tape.write(line + 'True')
else:
tape.write(line + 'False')
tape.write('\n')
for i, step in enumerate(E_step):
line = 'Step: ' + str(i) + ' ' + str(step)
tape.write(line + '\n')
### C Matrix
tape.write('[CM] \n')
tape.write('C AO times MO\n')
printmatrix(C, tape)
### D Matrix
tape.write('[DM] \n')
tape.write('D \n')
printmatrix(D, tape)
### D Matrix
tape.write('[NMO] \n')
tape.write('NMO \n')
printmatrix(mo_naturalorbital(D), tape)
### MO energies
tape.write('[MOE] \n')
tape.write('MOE \n')
for i, energ in enumerate(eigsysFockOp[0]):
tape.write(str(i) + ' ' + str(energ) + '\n')
### MO energies
tape.write('[INPUT] \n')
line = 'mol = ['
for i, coord in enumerate(xyz_atom):
line += '(' + str(charges[i]) + ','
line += '(' + str(coord[0]) + ',' + str(coord[1]) + ',' + str(
coord[2]) + ')),\n'
tape.write(line)
cont = 0
line = 'basis = ['
for i, ci in enumerate(contr_list):
line += '['
line += '(' + str(l[i][0]) + ',' + str(l[i][1]) + ',' + str(
l[i][2]) + '),'
for ii in range(ci):
line += str(alpha[cont]) + ',' + str(coef[i])
line += ',(' + str(xyz[i, 0]) + ',' + str(
xyz[i, 1]) + ',' + str(xyz[i, 2]) + ')],\n'
cont += 1
line += ']\n'
tape.write(line)
### Atom coordinates
tape.write('[Atoms]\n')
for i, coord in enumerate(xyz_atom):
line = select_atom.get(charges[i])
line += ' ' + str(i + 1) + ' ' + str(charges[i])
line += ' ' + str(coord[0]) + ' ' + str(coord[1]) + ' ' + str(
coord[2]) + '\n'
tape.write(line)
### Basis coordinates
for i, coord in enumerate(xyz):
line = 'XX'
line += ' ' + str(i + natoms + 1) + ' ' + str(0)
line += ' ' + str(coord[0]) + ' ' + str(coord[1]) + ' ' + str(
coord[2]) + '\n'
tape.write(line)
### Basis set
cont = 0
tape.write('[GTO]\n')
for i, ci in enumerate(contr_list):
tape.write(' ' + str(i + 1 + natoms) + ' 0\n')
if np.sum(l[i]) == 0:
tape.write(' s ' + str(ci) + ' 1.0 ' + str(l[i][0]) + ' ' +
str(l[i][1]) + ' ' + str(l[i][2]) + '\n')
else:
tape.write(' p ' + str(ci) + ' 1.0 ' + str(l[i][0]) + ' ' +
str(l[i][1]) + ' ' + str(l[i][2]) + '\n')
#tape.write(' p '+str(1)+' 1.0 '+ str(l[i])+'\n')
for ii in range(ci):
line = ' ' + str(alpha[cont]) + ' ' + str(coef[cont]) + '\n'
tape.write(line)
cont += 1
line = ' \n'
tape.write(line)
### MOs
tape.write('[MO]\n')
for j in range(nbasis):
tape.write(' Sym= None\n')
tape.write(' Ene= ' + str(eigsysFockOp[0][j]) + '\n')
tape.write(' Spin= Alpha\n')
if j > ne:
tape.write(' Occup= 0.0\n')
else:
tape.write(' Occup= 2.0\n')
for i in range(nbasis):
tape.write(str(i + 1 + natoms) + ' ' + str(C[i, j]) + '\n')
if status:
if write:
tape = open(name + '.molden', "w")
write_molden()
tape.close()
return E_elec + E_nuc
else:
print('E_elec: ' + str(E_elec) + '\n')
print('E_nuc: ' + str(E_nuc) + '\n')
print('E_tot: ' + str(E_nuc + E_elec) + '\n')
print('SCF DID NOT CONVERGED')
return 99999
return E_elec + E_nuc
