def trial(pair: Tuple[Type[Agent], Type[Environment]],
sign_bit: str,
num_trials: int = 100,
num_cycles: int = 10000,
seed: int = 1) -> Tuple[str, str, str, list]:
"""Trial agent in environment"""
agent_class, environment_class = pair[0], pair[1]
ag_name, env_name = agent_class.__name__, environment_class.__name__
rewards = []
sign = "positive" if sign_bit == "0" else "negative"
a, m = 16807, (pow(2, 31) - 1)
for trial_idx in range(num_trials):
total_reward = 0
environment = environment_class(sign_bit, seed=seed)
seed = (seed * a) % m
agent = agent_class(environment=environment, seed=seed)
seed = (seed * a) % m
observation = "0" * environment.observation_length
for cycle_idx in range(num_cycles):
action = agent.calculate_action(observation)
observation, reward = environment.calculate_percept(action)
total_reward += Utility.get_reward_from_bitstring(reward)
agent.train(reward)
total_reward /= num_cycles * environment_class.max_average_reward_per_cycle
rewards.append(total_reward)
return ag_name, env_name, sign, rewards
def calculate_action(self, observation: str) \
-> str:
"""Feed percept into nn and calculate best activations action. Returns action."""
action = ""
reward = -2
# calculate expected reward by trying every action
number_of_actions = pow(2, self.environment.action_length)
if self.seeded_rand_range(0, 10) == 0:
action_idx = self.seeded_rand_range(0, number_of_actions)
action_string = format(action_idx, 'b').zfill(self.environment.action_length)
nn_input = NNUtility.bitstr_to_narray(observation + action_string)
nn_output = self.nn.forward(nn_input)
action = action_string
self.activations = nn_output
else:
for action_idx in range(number_of_actions):
action_string = format(action_idx, 'b').zfill(self.environment.action_length)
nn_input = NNUtility.bitstr_to_narray(observation + action_string)
nn_output = self.nn.forward(nn_input)
reward_string = NNUtility.narray_to_bitstr(nn_output[1][-1])
action_reward = Utility.get_reward_from_bitstring(reward_string)
if action_reward == reward:
i = self.seeded_rand_range()
if i == 1:
action = action_string
self.activations = nn_output
else:
if action_reward > reward:
action = action_string
reward = action_reward
self.activations = nn_output
return action
def train(self, reward: str):
"""Add returned reward to statistic"""
reward_value = -1 * Utility.get_reward_from_bitstring(reward)
expected_reward = self.action_statistic[0]
action = self.action_statistic[1]
cnt = self.action_statistic[2]
new_expected_reward = (expected_reward * cnt + reward_value) / (cnt + 1)
heapq.heappush(self.table[self.observation], (new_expected_reward, action, cnt + 1))
def train(self, reward: str):
"""Add returned reward to statistic"""
self.cnt += 1
self.r = self.nr
self.nr = -1 * Utility.get_reward_from_bitstring(reward)
action_heap = self.table[self.sl_o][self.sl_a][self.l_o]
for idx in range(len(action_heap)):
if action_heap[idx][1] == self.l_a + self.a:
expected_reward = action_heap[idx][0]
reward = self.r + self.nr
cnt = action_heap[idx][2]
new_expected_reward = (expected_reward * cnt + reward) / (cnt + 1)
action_heap[idx] = (new_expected_reward, action_heap[idx][1], cnt + 1)
if expected_reward < reward:
Utility.heapq_siftup(action_heap, idx)
else:
Utility.heapq_siftdown(action_heap, 0, idx)
break