def get_response_content(fs):
# get the tree
tree = Newick.parse(fs.tree, Newick.NewickTree)
tree.assert_valid()
# get the alignment
try:
alignment = Fasta.Alignment(fs.fasta.splitlines())
alignment.force_nucleotide()
except Fasta.AlignmentError as e:
raise HandlingError(e)
# define the jukes cantor rate matrix
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(row_major_rate_matrix,
ordered_states)
# simulate the ancestral alignment
try:
alignment = PhyLikelihood.simulate_ancestral_alignment(
tree, alignment, rate_matrix_object)
except PhyLikelihood.SimulationError as e:
raise HandlingError(e)
# get the alignment string using an ordering defined by the tree
arr = []
for node in tree.preorder():
arr.append(alignment.get_fasta_sequence(node.name))
# return the response
return '\n'.join(arr) + '\n'
def __call__(self, X_logs):
"""
The vth entry of X corresponds to the log rate of the branch above v.
Return the quantity to be minimized (the neg log likelihood).
@param X: vector of branch rate logs
@return: negative log likelihood
"""
X = [math.exp(x) for x in X_logs]
B_subs = {}
for v_parent, v_child in self.R:
edge = frozenset([v_parent, v_child])
r = X[v_child]
t = self.B[edge]
B_subs[edge] = r * t
newick_string = FtreeIO.RBN_to_newick(self.R, B_subs, self.N_leaves)
tree = Newick.parse(newick_string, Newick.NewickTree)
# define the rate matrix object; horrible
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(
row_major_rate_matrix, ordered_states)
# get the log likelihood
ll = PhyLikelihood.get_log_likelihood(
tree, self.alignment, rate_matrix_object)
return -ll
def simulate_branch_path(tree, node):
"""
Simulate the nucleotide history on the path between a node and its parent.
This simulated path is conditional on known values at each node.
Purines are red; pyrimidines are blue.
A and T are brighter; G and C are darker.
@param tree: a SpatialTree with simulated nucleotides at each node
@param node: the node that defines the branch on which to simulate a history
"""
nucleotide_to_color = {
'A': 'FF4444',
'G': 'FF8888',
'T': '4444FF',
'C': '8888FF'
}
node.branch_color = nucleotide_to_color[node.state]
rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
initial_state = node.parent.state
terminal_state = node.state
states = 'ACGT'
events = None
while events is None:
events = PathSampler.get_nielsen_sample(initial_state, terminal_state,
states, node.blen, rate_matrix)
parent = node.parent
last_t = 0
for t, state in events:
new = SpatialTree.SpatialTreeNode()
new.name = node.name
new.state = state
new.branch_color = nucleotide_to_color[parent.state]
tree.insert_node(new, parent, node, (t - last_t) / float(node.blen))
last_t = t
parent = new
def get_response_content(fs):
# get the tree
tree = Newick.parse(fs.tree, Newick.NewickTree)
tree.assert_valid()
# get the alignment
try:
alignment = Fasta.Alignment(fs.fasta.splitlines())
alignment.force_nucleotide()
except Fasta.AlignmentError as e:
raise HandlingError(e)
# define the jukes cantor rate matrix
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(
row_major_rate_matrix, ordered_states)
# simulate the ancestral alignment
try:
alignment = PhyLikelihood.simulate_ancestral_alignment(
tree, alignment, rate_matrix_object)
except PhyLikelihood.SimulationError as e:
raise HandlingError(e)
# get the alignment string using an ordering defined by the tree
arr = []
for node in tree.preorder():
arr.append(alignment.get_fasta_sequence(node.name))
# return the response
return '\n'.join(arr) + '\n'
def simulate_branch_path(tree, node):
"""
Simulate the nucleotide history on the path between a node and its parent.
This simulated path is conditional on known values at each node.
Purines are red; pyrimidines are blue.
A and T are brighter; G and C are darker.
@param tree: a SpatialTree with simulated nucleotides at each node
@param node: the node that defines the branch on which to simulate a history
"""
nucleotide_to_color = {
'A':'FF4444', 'G':'FF8888', 'T':'4444FF', 'C':'8888FF'}
node.branch_color = nucleotide_to_color[node.state]
rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
initial_state = node.parent.state
terminal_state = node.state
states = 'ACGT'
events = None
while events is None:
events = PathSampler.get_nielsen_sample(
initial_state, terminal_state, states, node.blen, rate_matrix)
parent = node.parent
last_t = 0
for t, state in events:
new = SpatialTree.SpatialTreeNode()
new.name = node.name
new.state = state
new.branch_color = nucleotide_to_color[parent.state]
tree.insert_node(new, parent, node, (t - last_t) / float(node.blen))
last_t = t
parent = new
def demo_rejection_sampling():
path_length = 2
jukes_cantor_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
states = 'ACGT'
n = 100000
nielsen_event_count = 0
nielsen_path_count = 0
nielsen_first_time_sum = 0
nielsen_dwell = dict((c, 0) for c in states)
rejection_event_count = 0
rejection_path_count = 0
rejection_first_time_sum = 0
rejection_dwell = dict((c, 0) for c in states)
for i in range(n):
initial_state = 'A'
terminal_state = 'C'
events = get_rejection_sample(initial_state, terminal_state, states, path_length, jukes_cantor_rate_matrix)
if events is not None:
assert events
rejection_path_count += 1
rejection_event_count += len(events)
t, state = events[0]
rejection_first_time_sum += t
extended = [(0, initial_state)] + events + [(path_length, terminal_state)]
for (t0, state0), (t1, state1) in zip(extended[:-1], extended[1:]):
rejection_dwell[state0] += t1 - t0
events = get_nielsen_sample(initial_state, terminal_state, states, path_length, jukes_cantor_rate_matrix)
if events is not None:
assert events
nielsen_path_count += 1
nielsen_event_count += len(events)
t, state = events[0]
nielsen_first_time_sum += t
extended = [(0, initial_state)] + events + [(path_length, terminal_state)]
for (t0, state0), (t1, state1) in zip(extended[:-1], extended[1:]):
nielsen_dwell[state0] += t1 - t0
expected_fraction = RateMatrix.get_jukes_cantor_transition_matrix(path_length)[(initial_state, terminal_state)]
print 'testing the rejection sampling:'
print 'expected fraction:', expected_fraction
print 'observed fraction:', rejection_path_count / float(n)
print 'comparing rejection sampling and nielsen sampling:'
rejection_method_fraction = rejection_event_count / float(rejection_path_count)
nielsen_method_fraction = nielsen_event_count / float(nielsen_path_count)
print 'rejection method fraction:', rejection_method_fraction
print 'nielsen method fraction:', nielsen_method_fraction
print 'comparing time of first event:'
print 'rejection method first event time mean:', rejection_first_time_sum / float(rejection_path_count)
print 'nielsen method first event time mean:', nielsen_first_time_sum / float(nielsen_path_count)
print 'comparing the duration spent in each state:'
print 'rejection:'
for state, t in rejection_dwell.items():
print '\t%s: %f' % (state, t/float(rejection_path_count))
print 'nielsen:'
for state, t in nielsen_dwell.items():
print '\t%s: %f' % (state, t/float(nielsen_path_count))
def get_response_content(fs):
# get a properly formatted newick tree with branch lengths
tree = Newick.parse(fs.tree, SpatialTree.SpatialTree)
tree.assert_valid()
if tree.has_negative_branch_lengths():
msg = 'drawing a tree with negative branch lengths is not implemented'
raise HandlingError(msg)
tree.add_branch_lengths()
# get the dictionary mapping the branch name to the nucleotide
name_to_nucleotide = {}
# parse the column string
for line in iterutils.stripped_lines(fs.column.splitlines()):
name_string, nucleotide_string = SnippetUtil.get_state_value_pair(line)
if nucleotide_string not in list('acgtACGT'):
msg = '"%s" is not a valid nucleotide' % nucleotide_string
raise HandlingError(msg)
nucleotide_string = nucleotide_string.upper()
if name_string in name_to_nucleotide:
raise HandlingError('the name "%s" was duplicated' % name_string)
name_to_nucleotide[name_string] = nucleotide_string
# augment the tips with the nucleotide letters
for name, nucleotide in name_to_nucleotide.items():
try:
node = tree.get_unique_node(name)
except Newick.NewickSearchError as e:
raise HandlingError(e)
if node.children:
msg = 'constraints on internal nodes are not implemented'
raise HandlingError(msg)
node.state = nucleotide
# get the Jukes-Cantor rate matrix object
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(row_major_rate_matrix,
ordered_states)
# simulate the ancestral nucleotides
rate_matrix_object.simulate_ancestral_states(tree)
# simulate a path on each branch
# this breaks up the branch into a linear sequence of nodes and adds color
for node in tree.gen_non_root_nodes():
simulate_branch_path(tree, node)
# do the layout
EqualArcLayout.do_layout(tree)
# draw the image
try:
ext = Form.g_imageformat_to_ext[fs.imageformat]
return DrawTreeImage.get_tree_image(tree, (640, 480), ext)
except CairoUtil.CairoUtilError as e:
raise HandlingError(e)
def get_response_content(fs):
# get a properly formatted newick tree with branch lengths
tree = Newick.parse(fs.tree, SpatialTree.SpatialTree)
tree.assert_valid()
if tree.has_negative_branch_lengths():
msg = 'drawing a tree with negative branch lengths is not implemented'
raise HandlingError(msg)
tree.add_branch_lengths()
# get the dictionary mapping the branch name to the nucleotide
name_to_nucleotide = {}
# parse the column string
for line in iterutils.stripped_lines(fs.column.splitlines()):
name_string, nucleotide_string = SnippetUtil.get_state_value_pair(line)
if nucleotide_string not in list('acgtACGT'):
msg = '"%s" is not a valid nucleotide' % nucleotide_string
raise HandlingError(msg)
nucleotide_string = nucleotide_string.upper()
if name_string in name_to_nucleotide:
raise HandlingError('the name "%s" was duplicated' % name_string)
name_to_nucleotide[name_string] = nucleotide_string
# augment the tips with the nucleotide letters
for name, nucleotide in name_to_nucleotide.items():
try:
node = tree.get_unique_node(name)
except Newick.NewickSearchError as e:
raise HandlingError(e)
if node.children:
msg = 'constraints on internal nodes are not implemented'
raise HandlingError(msg)
node.state = nucleotide
# get the Jukes-Cantor rate matrix object
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(
row_major_rate_matrix, ordered_states)
# simulate the ancestral nucleotides
rate_matrix_object.simulate_ancestral_states(tree)
# simulate a path on each branch
# this breaks up the branch into a linear sequence of nodes and adds color
for node in tree.gen_non_root_nodes():
simulate_branch_path(tree, node)
# do the layout
EqualArcLayout.do_layout(tree)
# draw the image
try:
ext = Form.g_imageformat_to_ext[fs.imageformat]
return DrawTreeImage.get_tree_image(tree, (640, 480), ext)
except CairoUtil.CairoUtilError as e:
raise HandlingError(e)
def test_simulation(self):
tree_string = '(((Human:0.1, Chimpanzee:0.2)to-chimp:0.8, Gorilla:0.3)to-gorilla:0.7, Orangutan:0.4, Gibbon:0.5)all;'
# Parse the example tree.
tree = Newick.parse(tree_string, Newick.NewickTree)
tree.assert_valid()
# Get header and sequence pairs.
alignment = Fasta.Alignment(StringIO(Fasta.brown_example_alignment))
# Get the Jukes-Cantor rate matrix object.
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(row_major_rate_matrix, ordered_states)
# Simulate ancestral states.
simulated_alignment = simulate_ancestral_alignment(tree, alignment, rate_matrix_object)
def gen_distance_matrices(self, count, max_steps):
"""
Yield (ordered sequence list, distance matrix) pairs .
The generator will stop if it sees that it cannot meet its goal
in the allotted number of steps.
@param count: the requested number of distance matrices
@param max_steps: an upper bound on the allowed number of steps
"""
# define the jukes cantor rate matrix
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
model = RateMatrix.RateMatrix(row_major_rate_matrix, ordered_states)
# record the requested number of samples
self.requested_matrix_count = count
# do some rejection sampling
while True:
if self.get_complexity() >= max_steps:
break
if self.accepted_sample_count >= count:
break
# simulate an alignment from the tree
alignment = PhyLikelihood.simulate_alignment(
self.tree, model, self.sequence_length)
# extract the ordered list of sequences from the alignment object
name_to_sequence = dict(zip(alignment.headers,
alignment.sequences))
sequence_list = [
name_to_sequence[name] for name in self.ordered_names
]
# get the estimated distance matrix
distance_matrix = JC69.get_ML_distance_matrix(sequence_list)
# look for degeneracies
has_zero_off_diagonal = False
has_inf_off_diagonal = False
for i, row in enumerate(distance_matrix):
for j, value in enumerate(row):
if i != j:
if value == 0.0:
has_zero_off_diagonal = True
if value == float('inf'):
has_inf_off_diagonal = True
if has_zero_off_diagonal:
self.rejected_zero_sample_count += 1
elif has_inf_off_diagonal:
self.rejected_inf_sample_count += 1
else:
self.accepted_sample_count += 1
yield sequence_list, distance_matrix
def test_likelihood(self):
# Parse the example tree.
tree_string = Newick.brown_example_tree
tree = Newick.parse(tree_string, Newick.NewickTree)
tree.assert_valid()
# Get header and sequence pairs.
alignment = Fasta.Alignment(StringIO(Fasta.brown_example_alignment))
# Get the Jukes-Cantor rate matrix object.
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(row_major_rate_matrix, ordered_states)
# Calculate the log likelihood.
log_likelihood = get_log_likelihood(tree, alignment, rate_matrix_object)
self.assertAlmostEqual(log_likelihood, -4146.26547208)
def test_jukes_cantor_rejection(self):
path_length = 1
jukes_cantor_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
states = 'ACGT'
n = 200
observed = 0
for i in range(n):
events = get_rejection_sample('A', 'C', states, path_length, jukes_cantor_rate_matrix)
if events is not None:
observed += 1
p = RateMatrix.get_jukes_cantor_transition_matrix(path_length)[('A', 'C')]
expected = n*p
variance = n*p*(1-p)
errstr = 'observed: %f expected: %f' % (observed, expected)
self.failUnless(abs(observed - expected) < 3*math.sqrt(variance), errstr)
def gen_distance_matrices(self, count, max_steps):
"""
Yield (ordered sequence list, distance matrix) pairs .
The generator will stop if it sees that it cannot meet its goal
in the allotted number of steps.
@param count: the requested number of distance matrices
@param max_steps: an upper bound on the allowed number of steps
"""
# define the jukes cantor rate matrix
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
model = RateMatrix.RateMatrix(row_major_rate_matrix, ordered_states)
# record the requested number of samples
self.requested_matrix_count = count
# do some rejection sampling
while True:
if self.get_complexity() >= max_steps:
break
if self.accepted_sample_count >= count:
break
# simulate an alignment from the tree
alignment = PhyLikelihood.simulate_alignment(
self.tree, model, self.sequence_length)
# extract the ordered list of sequences from the alignment object
name_to_sequence = dict(zip(alignment.headers, alignment.sequences))
sequence_list = [name_to_sequence[name]
for name in self.ordered_names]
# get the estimated distance matrix
distance_matrix = JC69.get_ML_distance_matrix(sequence_list)
# look for degeneracies
has_zero_off_diagonal = False
has_inf_off_diagonal = False
for i, row in enumerate(distance_matrix):
for j, value in enumerate(row):
if i != j:
if value == 0.0:
has_zero_off_diagonal = True
if value == float('inf'):
has_inf_off_diagonal = True
if has_zero_off_diagonal:
self.rejected_zero_sample_count += 1
elif has_inf_off_diagonal:
self.rejected_inf_sample_count += 1
else:
self.accepted_sample_count += 1
yield sequence_list, distance_matrix
def get_response_content(fs):
# get the tree
tree = Newick.parse(fs.tree, Newick.NewickTree)
tree.assert_valid()
# get the alignment
try:
alignment = Fasta.Alignment(fs.fasta.splitlines())
alignment.force_nucleotide()
except Fasta.AlignmentError as e:
raise HandlingError(e)
# get the log likelihood
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(
row_major_rate_matrix, ordered_states)
log_likelihood = PhyLikelihood.get_log_likelihood(
tree, alignment, rate_matrix_object)
# return the response
return str(log_likelihood) + '\n'
def get_response_content(fs):
# init the response and get the user variables
out = StringIO()
nleaves = fs.nleaves
nvertices = nleaves * 2 - 1
nbranches = nvertices - 1
nsites = fs.nsites
# sample the coalescent tree with timelike branch lengths
R, B = kingman.sample(fs.nleaves)
r = Ftree.R_to_root(R)
# get the leaf vertex names
N = dict(zip(range(nleaves), string.uppercase[:nleaves]))
N_leaves = dict(N)
# get the internal vertex names
v_to_leaves = R_to_v_to_leaves(R)
for v, leaves in sorted(v_to_leaves.items()):
if len(leaves) > 1:
N[v] = ''.join(sorted(N[leaf] for leaf in leaves))
# get vertex ages
v_to_age = kingman.RB_to_v_to_age(R, B)
# sample the rates on the branches
b_to_rate = sample_b_to_rate(R)
xycorr = get_correlation(R, b_to_rate)
# define B_subs in terms of substitutions instead of time
B_subs = dict((p, t * b_to_rate[p]) for p, t in B.items())
# sample the alignment
v_to_seq = sample_v_to_seq(R, B_subs, nsites)
# get the log likelihood; this is kind of horrible
pairs = [(N[v], ''.join(v_to_seq[v])) for v in range(nleaves)]
headers, sequences = zip(*pairs)
alignment = Fasta.create_alignment(headers, sequences)
newick_string = FtreeIO.RBN_to_newick(R, B_subs, N_leaves)
tree = Newick.parse(newick_string, Newick.NewickTree)
dictionary_rate_matrix = RateMatrix.get_jukes_cantor_rate_matrix()
ordered_states = list('ACGT')
row_major_rate_matrix = MatrixUtil.dict_to_row_major(
dictionary_rate_matrix, ordered_states, ordered_states)
rate_matrix_object = RateMatrix.RateMatrix(
row_major_rate_matrix, ordered_states)
ll = PhyLikelihood.get_log_likelihood(
tree, alignment, rate_matrix_object)
# get ll when rates are all 1.0
newick_string = FtreeIO.RBN_to_newick(R, B, N_leaves)
tree = Newick.parse(newick_string, Newick.NewickTree)
ll_unity = PhyLikelihood.get_log_likelihood(
tree, alignment, rate_matrix_object)
# get ll when rates are numerically optimized
# TODO incorporate the result into the xml file
# TODO speed up the likelihood evaluation (beagle? C module?)
#f = Opt(R, B, N_leaves, alignment)
#X_logs = [0.0] * nbranches
#result = scipy.optimize.fmin(f, X_logs, full_output=True)
#print result
#
print >> out, '<?xml version="1.0"?>'
print >> out, '<beast>'
print >> out
print >> out, '<!-- actual rate autocorrelation', xycorr, '-->'
print >> out, '<!-- actual root height', v_to_age[r], '-->'
print >> out, '<!-- actual log likelihood', ll, '-->'
print >> out, '<!-- ll if rates were unity', ll_unity, '-->'
print >> out
print >> out, '<!--'
print >> out, 'predefine the taxa as in'
print >> out, 'http://beast.bio.ed.ac.uk/Introduction_to_XML_format'
print >> out, '-->'
print >> out, get_leaf_taxon_defn(list(string.uppercase[:nleaves]))
print >> out
print >> out, '<!--'
print >> out, 'define the alignment as in'
print >> out, 'http://beast.bio.ed.ac.uk/Introduction_to_XML_format'
print >> out, '-->'
print >> out, get_alignment_defn(leaves, N, v_to_seq)
print >> out
print >> out, '<!--'
print >> out, 'specify the starting tree as in'
print >> out, 'http://beast.bio.ed.ac.uk/Tutorial_4'
print >> out, '-->'
print >> out, get_starting_tree_defn(R, B, N_leaves)
print >> out
print >> out, '<!--'
print >> out, 'connect the tree model as in'
print >> out, 'http://beast.bio.ed.ac.uk/Tutorial_4'
print >> out, '-->'
print >> out, g_tree_model_defn
print >> out
print >> out, g_uncorrelated_relaxed_clock_info
print >> out
"""
print >> out, '<!--'
print >> out, 'create a list of taxa for which to constrain the mrca as in'
print >> out, 'http://beast.bio.ed.ac.uk/Tutorial_3.1'
print >> out, '-->'
for v, leaves in sorted(v_to_leaves.items()):
if len(leaves) > 1:
print >> out, get_mrca_subset_defn(N, v, leaves)
print >> out
print >> out, '<!--'
print >> out, 'create a tmrcaStatistic that will record the height as in'
print >> out, 'http://beast.bio.ed.ac.uk/Tutorial_3.1'
print >> out, '-->'
for v, leaves in sorted(v_to_leaves.items()):
if len(leaves) > 1:
print >> out, get_mrca_stat_defn(N[v])
"""
print >> out
print >> out, g_likelihood_info
print >> out
print >> out, '<!--'
print >> out, 'run the mcmc'
print >> out, 'http://beast.bio.ed.ac.uk/Tutorial_3.1'
print >> out, '-->'
print >> out, get_mcmc_defn(v_to_leaves, v_to_age, N)
print >> out
print >> out, '</beast>'
# return the response
return out.getvalue()
