def look_ahead(self):
"""
Look-ahead step in the current state of the environment to select the real best next action
:return: best action to take in this state
"""
options = [
align_progressive(self.env.permutation + [i], self.sequences,
self.env.align_table).score()[self.score]
if v == 1 else 0 for i, v in enumerate(self.env.available)
]
return options.index(max(options))
def rollout(self, step):
"""
perform a rollout starting/continuing in this node/state
:param step: number of games played before
:return: the score reached in the rollout
"""
# i this is a leaf return the represented score
if self.is_leaf:
return self.scoring
# if this node has to be expanded, expand and continue
if self.to_expand:
self.expand()
# if this node is already expanded or has just been expanded select the
if self.is_expanded:
# select a node according to the tree-policy and pass the rollout to this node
node = select_child(self.children)
rollout_score = node.rollout(step)
# update the children of this node according to the received result
for c in self.children:
if c != node:
c.update_u(step)
else:
node.visit(rollout_score, step)
# if this is the first explicit, by the tree-policy selected visit after creation, perform a rollout
else:
rollout_score = 0
for _ in range(self.mcts_agent.rollouts):
tmp_state = self.state.copy()
self.to_expand = True
possible_actions = list(
range(-1, self.num_seqs - 1) if self.mcts_agent.
refinement else set(range(self.num_seqs)) - set(tmp_state))
# append all possible actions in an proper way so that the result is not 0
while len(tmp_state) < self.num_seqs:
action = np.random.choice(possible_actions)
tmp_state = tmp_state + [action]
if not self.mcts_agent.refinement:
possible_actions.remove(action)
# compute the alignments score
if len(tmp_state) == len(set(tmp_state)):
if self.mcts_agent.refinement:
tmp_score = align_iterative(
tmp_state, self.mcts_agent.profile,
self.mcts_agent.align_table).score()[self.score]
else:
tmp_score = align_progressive(
tmp_state, self.mcts_agent.sequences,
self.mcts_agent.align_table).score()[self.score]
if self.mcts_agent.adjust:
rollout_score += self.mcts_agent.adjustment(tmp_score)
return rollout_score
def estimate_min(self):
"""
Estimate minimal score for problem by taking the minimal score of 10 random rollouts form the initial node
:return: estimated minimal score
"""
return min([
align_progressive(
list(
np.random.permutation(
list(
range(self.num_seqs -
(1 if self.refinement else 0))))),
self.sequences, self.align_table).score()[self.score]
for _ in range(10)
]) - 1
def __init__(self, state, num_seqs, score, mcts_agent):
"""
initialize the node
:param state: state represented by the node
:param num_seqs: number of sequences to align in total
:param score: score to optimize for
:param mcts_agent: parent agent, used to access algorithmic fields like
the number of rollouts, the sequences, the UCB-parameter c and the align_table
"""
self.state = state
self.num_seqs = num_seqs
self.children = []
self.score = score
self.mcts_agent = mcts_agent
# search parameters
self.n = 0
self.u = 0
self.v = 0
self.total_score = 0
# initialize fields to track state of the node (leaf / (tb.) expanded)
self.is_leaf = self.num_seqs == len(self.state)
self.to_expand = False
self.is_expanded = False
# compute the score of the alignment represented by this node if this node is a leaf that cannot be expanded
self.scoring = 0
if self.is_leaf and len(set(self.state)) == self.num_seqs:
if self.mcts_agent.refinement:
self.scoring = align_iterative(
self.state,
self.mcts_agent.profile,
self.mcts_agent.align_table,
).score()[self.score]
else:
self.scoring = align_progressive(
self.state, self.mcts_agent.sequences,
self.mcts_agent.align_table).score()[self.score]
if self.mcts_agent.adjust:
self.scoring = self.mcts_agent.adjustment(self.scoring)
def initialize_benchmark(b_id, best):
"""
Initialize the execution of any agents on the benchmarks by creating necessary fields needed for search and training
:param b_id: benchmark-id to train on
:param best: best result from previous training runs on that benchmark
:return: - the name of the benchmark used for evaluation tables at the end
- the sequences of the benchmark as list of stings
- the names according to the sequences in the same order ( = permutation [0,1,...,n-1,n])
- basic data of the benchmark-sequences, namely type, count and average length
- basic comparison of available results for this benchmark
- best result from previous run on this benchmark
"""
'''
Things like comparison of different agents, best-marks on benchmarks and align-tables are stored in such nested
HashMaps and are accessed and modified according to the actually used configuration and its scoring and aligning
The benchmark_best contains the best results on a benchmark for each alignment setting. Such an refinement-tuple
consists of the Profile from the iterative aligning, the according iterative permutation, the permutation for the
starting alignment and the configuration that led to the optimal alignment
'''
comparison = {
"SP": {
"Refinement": {},
"Progressive": {}
},
"CS": {
"Refinement": {},
"Progressive": {}
}
}
benchmark_best = {
"SP": {
"Refinement": (Profile([]), None, None, None),
"Progressive": (Profile([]), None, None)
},
"CS": {
"Refinement": (Profile([]), None, None, None),
"Progressive": (Profile([]), None, None)
}
}
# if the benchmark is of this work and known because it is used while development, extract data from constants
if isinstance(b_id, int):
name, b, seqs_file = names[b_id], benchmarks[b_id], seq_files[b_id]
sequences, sequence_names = read_fasta_data(seqs_file)
# Insert the base-data for each setting, this is the left-hand side of the tables outputted at the end
comparison["SP"]["Progressive"] = {
RL: b[0:2],
DRL: b[2:4],
CLUSTALW: b[4:6],
MAFFT: b[6:8],
MUSCLE: b[8:10]
}
comparison["CS"]["Progressive"] = {
RL: b[0:2],
DRL: b[2:4],
CLUSTALW: b[4:6],
MAFFT: b[6:8],
MUSCLE: b[8:10]
}
comparison["SP"]["Refinement"] = {
RL: b[0:2],
DRL: b[2:4],
CLUSTALW: b[4:6],
MAFFT: b[6:8],
MUSCLE: b[8:10]
}
comparison["CS"]["Refinement"] = {
RL: b[0:2],
DRL: b[2:4],
CLUSTALW: b[4:6],
MAFFT: b[6:8],
MUSCLE: b[8:10]
}
base_data = (b_id, types[b_id], sizes[b_id])
# else read in the benchmark sequences and compute the base data
else:
name, (sequences, sequence_names
) = os.path.basename(b_id).split(".")[0], read_fasta_data(b_id)
# again the base data as left-hand side of the output tables, but here with zeros as they are not performed
comparison["SP"]["Progressive"] = {
RL: (0, 0),
DRL: (0, 0),
CLUSTALW: (0, 0),
MAFFT: (0, 0),
MUSCLE: (0, 0)
}
comparison["CS"]["Progressive"] = {
RL: (0, 0),
DRL: (0, 0),
CLUSTALW: (0, 0),
MAFFT: (0, 0),
MUSCLE: (0, 0)
}
comparison["SP"]["Refinement"] = {
RL: (0, 0),
DRL: (0, 0),
CLUSTALW: (0, 0),
MAFFT: (0, 0),
MUSCLE: (0, 0)
}
comparison["CS"]["Refinement"] = {
RL: (0, 0),
DRL: (0, 0),
CLUSTALW: (0, 0),
MAFFT: (0, 0),
MUSCLE: (0, 0)
}
base_data = (b_id, get_sequence_type(sequences),
get_sequence_size(sequences))
'''
Find and fill in the best alignments per optimization setting that can be found for the individual benchmark in the
store of best alignments. If it is not known, the (Profile([]), None, None) tuple remains
'''
if name in best["SP"]["Progressive"]:
tmp = best["SP"]["Progressive"][name]
benchmark_best["SP"]["Progressive"] = (align_progressive(
tmp["Permutation"],
sequences), tmp["Permutation"], from_dict(tmp["Configuration"]))
if name in best["CS"]["Progressive"]:
tmp = best["CS"]["Progressive"][name]
benchmark_best["CS"]["Progressive"] = (align_progressive(
tmp["Permutation"],
sequences), tmp["Permutation"], from_dict(tmp["Configuration"]))
if name in best["SP"]["Refinement"]:
tmp = best["SP"]["Refinement"][name]
benchmark_best["SP"]["Refinement"] = \
(align_iterative(tmp["Permutation"], align_progressive(tmp["BasePermutation"], sequences)),
tmp["Permutation"], tmp["BasePermutation"], from_dict(tmp["Configuration"]))
if name in best["CS"]["Refinement"]:
tmp = best["CS"]["Refinement"][name]
benchmark_best["CS"]["Refinement"] = \
(align_iterative(tmp["Permutation"], align_progressive(tmp["BasePermutation"], sequences)),
tmp["Permutation"], tmp["BasePermutation"], from_dict(tmp["Configuration"]))
'''
insert the baseline (aka score of best progressive alignment) into the results-table to see from which value the
agent stared its alignment and to be able to argue on whether the alignment has been improved or not
'''
comparison["SP"]["Refinement"][TABLE_AGENT] = (
*(benchmark_best["SP"]["Progressive"][0].score()[0:2]), 0, 0)
comparison["CS"]["Refinement"][TABLE_AGENT] = (
*(benchmark_best["CS"]["Progressive"][0].score()[0:2]), 0, 0)
return name, sequences, sequence_names, base_data, comparison, benchmark_best
def run_agent(config, sequences, align_table, name, best_score, updating,
total, individual, data_file):
"""
run the by config specified agent on the sequences and use the given align-table as util to speed up th alignment
:param config: configuration of the agent to train
:param sequences: sequences of the benchmark to run the agent on
:param align_table: table to use for alignment-speedup
:param name: name of the benchmark, needed for the reports after training
:param best_score: best_score ever reached on this benchmark
:param updating: flag indicating to update the json file
:param total: total number of trainings performed in this run
:param individual: flag to indicate the use of a fresh align-table
:param data_file: file to store all data about the computations
:return: - the message to print and to sent via Telegram,
- the score reached by the agent,
- the extended align_table with probably new alignments,
- the profile computed by this algorithm that leads to the result score and
- the permutation of the input sequences to get the resulting profile
"""
print(F"train {config.name} agent result on {name}")
global number
if config.id == MCTS_AGENT:
agent = MCTSAgent(sequences,
simulations=config.simulations,
rollouts=config.rollouts,
c=config.c,
score=config.score,
refinement=config.refinement,
console=config.print_console,
adjust=config.adjust)
if not individual:
agent.set_align_table(align_table)
# compute the alignment using UCT-MCTS to find the most promising sequence
env = RefinementWrapper(
sequences, agent,
config.score) if config.refinement else AlignmentWrapper(
sequences, agent)
try:
start = time.time()
_, permutation, profile, _ = env.run()
end = time.time()
runtime = end - start
except:
print("MCTS-Agent crashed! Return standard results.")
permutation = [] if config.refinement else list(
range(len(sequences)))
profile = sequences if config.refinement else align_progressive(
permutation, sequences)
runtime = 0
align_table = agent.get_align_table()
print("Align-Table-Stats:", align_table.stats)
# extract the score
score = profile.score() if profile is not None else (0, 0)
scoring = (score[0], score[1], "-", "-")
# create the results-message
mode = "Refinement" if config.refinement else "Progressive"
optimizing = "C" if config.score == C_SCORE else "SP"
message = F"MCTS algorithm result on {name}:\n" \
F"\tMode: {mode}, Optimizing: {optimizing}-Score\n" \
F"\tProgress: {number}/{total} ({round((number / total) * 100, 2)}%)\n" \
F"\tRuntime: {runtime} seconds\n" \
F"\tbest Alignment found: {score[0]}, {round(score[1], 2)}\n"
data_file.write(
F"MCTS\t{name}\t{mode}\t{optimizing}\t{runtime}\t{score[0]}\t{round(score[1], 2)}\t["
+ ";".join([str(p) for p in permutation]) + "]\t[" +
";".join([str(s) for s in align_table.stats]) + "]\n")
else:
# if not MCTS agent, create one of the other agents
if config.id == TABLE_AGENT:
agent, agent_trainer = create_table_agent(sequences, config)
elif config.id == VALUE_AGENT:
agent, agent_trainer = create_value_agent(sequences, config)
elif config.id == POLICY_AGENT:
agent, agent_trainer = create_policy_agent(sequences, config)
elif config.id == ACTOR_CRITIC_AGENT:
agent, agent_trainer = create_actor_critic_agent(sequences, config)
else:
agent, agent_trainer = create_alpha_zero_agent(sequences, config)
# train the agent ...
if not individual:
agent_trainer.set_align_table(align_table)
try:
_, _, _, runtime = agent_trainer.run(config.print_console,
config.print_graph)
align_table = agent_trainer.get_align_table()
print("Align-Table-Stats:", align_table.stats)
# ... and compute the final alignment
(profile, permutation), (reward, _, _,
_) = agent_trainer.evaluate_training()
except:
print("Error occurred in training of ")
permutation = list(range(len(sequences)))
profile = align_progressive(permutation, sequences, align_table)
reward = 0, 0
runtime = 0
# extract the score
score = profile.score() if profile is not None else (0, 0)
scoring = (score[0], score[1], reward[0], reward[1])
# create the results-message
mode = "Refinement" if config.refinement else "Progressive"
optimizing = "C" if config.score == C_SCORE else "SP"
message = F"{agent.name()} algorithm result on {name}:\n" \
F"\tMode: {mode}, Optimizing: {optimizing}-Score\n" \
F"\tProgress: {number}/{total} ({round((number / total) * 100, 2)}%)\n" \
F"\tRuntime: {runtime} seconds\n" \
F"\tbest Alignment found: {score[0]}, {round(score[1], 2)}\n" \
F"\tlast Alignment found: {reward[0]}, {round(reward[1], 2)}"
data_file.write(
F"{agent.name}\t{name}\t{mode}\t{optimizing}\t{runtime}\t{score[0]}\t{round(score[1], 2)}\t["
+ ";".join([str(p) for p in permutation]) + "]\t[" +
";".join([str(s) for s in align_table.stats]) + "]\n")
# increase counter for configs that have been tested
number += 1
# check, whether older alignment has been improved
if updating and (score[config.score], score[1 - config.score]) > \
(best_score[config.score], best_score[1 - config.score]):
message = "NEW BEST ALIGNMENT FOUND!\n" + message
# print the message of the results and notify via telegram
print(message)
if config.notification:
notify(message)
return message, scoring, align_table, profile, permutation
def test_align_progressive_2(self):
seqs = ["ctattg", "ctaccg", "ctatgt"]
profile = align_progressive([1, 2, 0], seqs)
self.assertEqual(["ctaccg-", "cta-tgt", "ctattg-"],
profile.get_sequences())
def test_align_progressive_1(self):
seqs = ["ctattg", "ctaccg", "ctatgt"]
profile = align_progressive([2, 0, 1], seqs)
self.assertEqual(["ctatgt-", "ctat-tg", "ctac-cg"],
profile.get_sequences())