def worker(worker_id, master_end, worker_end, game_params, map_name,
obs_proc_params, action_dict):
master_end.close() # Forbid worker to use the master end for messaging
np.random.seed() # sets random seed for the environment
env = init_game(game_params,
map_name,
random_seed=np.random.randint(10000))
op = ObsProcesser(**obs_proc_params)
while True:
cmd, data = worker_end.recv()
if cmd == 'step':
obs = env.step([data])
state_trg_dict, _ = op.get_state(
obs) #returns (state_dict, names_dict)
state_trg = merge_screen_and_minimap(state_trg_dict)
reward = obs[0].reward
done = obs[0].last()
# Always bootstrap when episode finishes (in MoveToBeacon there is no real end)
if done:
bootstrap = True
else:
bootstrap = False
# state_trg is the state used as next state for the update
# state is the new state used to decide the next action
# (different if the episode ends and another one begins)
if done:
obs = reset_and_skip_first_frame(env)
state_dict, _ = op.get_state(
obs) # returns (state_dict, names_dict)
state = merge_screen_and_minimap(state_dict)
else:
state = state_trg
available_actions = obs[0].observation.available_actions
action_mask = get_action_mask(available_actions, action_dict)
worker_end.send(
(state, reward, done, bootstrap, state_trg, action_mask))
elif cmd == 'reset':
obs = reset_and_skip_first_frame(env)
state_dict, _ = op.get_state(
obs) # returns (state_dict, names_dict)
state = merge_screen_and_minimap(state_dict)
available_actions = obs[0].observation.available_actions
action_mask = get_action_mask(available_actions, action_dict)
worker_end.send((state, action_mask))
elif cmd == 'close':
worker_end.close()
break
else:
raise NotImplementedError
def test(step_idx, agent, test_env, process_ID, op, action_table, num_test, save_path):
score = 0.0
done = False
for _ in range(num_test):
obs = reset_and_skip_first_frame(test_env)
s_dict, _ = op.get_state(obs)
spatial, player = merge_screen_and_minimap(s_dict)
s = {"spatial":spatial, "player":player}
for k in s.keys():
s[k] = s[k][np.newaxis, ...] # add batch dim
available_actions = obs[0].observation.available_actions
a_mask = get_action_mask(available_actions, action_table)[np.newaxis, ...] # add batch dim
while not done:
a, log_prob, probs = agent.step(s, a_mask)
obs = test_env.step(a)
s_prime_dict, _ = op.get_state(obs)
spatial, player = merge_screen_and_minimap(s_prime_dict)
s_prime = {"spatial":spatial, "player":player}
for k in s_prime.keys():
s_prime[k] = s_prime[k][np.newaxis, ...] # add batch dim
reward = obs[0].reward
done = obs[0].last()
available_actions = obs[0].observation.available_actions
a_mask = get_action_mask(available_actions, action_table)[np.newaxis, ...] # add batch dim
s = s_prime
score += reward
done = False
with open(save_path+'/Logging/'+process_ID+'.txt', 'a+') as f:
print(f"{step_idx},{score/num_test:.1f}", file=f)
return score/num_test
def inspection_test(step_idx, agent, test_env, process_ID, op, action_dict):
inspector = InspectionDict(step_idx, process_ID, action_dict, test_env)
obs = reset_and_skip_first_frame(test_env)
s_dict, _ = op.get_state(obs)
s = merge_screen_and_minimap(s_dict)
s = s[np.newaxis, ...] # add batch dim
available_actions = obs[0].observation.available_actions
a_mask = get_action_mask(available_actions,
action_dict)[np.newaxis, ...] # add batch dim
done = False
G = 0.0
# list used for update
s_lst, r_lst, done_lst, bootstrap_lst, s_trg_lst = list(), list(), list(
), list(), list()
log_probs = []
entropies = []
while not done:
a, log_prob, entropy = inspection_step(agent, inspector, s, a_mask)
log_probs.append(log_prob)
entropies.append(entropy)
obs = test_env.step(a)
s_prime_dict, _ = op.get_state(obs)
s_prime = merge_screen_and_minimap(s_prime_dict)
s_prime = s_prime[np.newaxis, ...] # add batch dim
reward = obs[0].reward
done = obs[0].last()
available_actions = obs[0].observation.available_actions
a_mask = get_action_mask(available_actions,
action_dict)[np.newaxis, ...] # add batch dim
if done:
bootstrap = True
else:
bootstrap = False
inspector.dict['state_traj'].append(s)
s_lst.append(s)
r_lst.append(reward)
done_lst.append(done)
bootstrap_lst.append(bootstrap)
s_trg_lst.append(s_prime)
s = s_prime
G += reward
inspector.dict['rewards'] = r_lst
s_lst = np.array(s_lst).transpose(1, 0, 2, 3, 4)
r_lst = np.array(r_lst).reshape(1, -1)
done_lst = np.array(done_lst).reshape(1, -1)
bootstrap_lst = np.array(bootstrap_lst).reshape(1, -1)
s_trg_lst = np.array(s_trg_lst).transpose(1, 0, 2, 3, 4)
update_dict = inspection_update(agent, r_lst, log_probs, entropies, s_lst,
done_lst, bootstrap_lst, s_trg_lst)
inspector.store_update(update_dict)
return inspector