def test(test_episodes):
tf.reset_default_graph()
policy_nn = SupervisedPolicy()
f = open(relationPath)
test_data = f.readlines()
f.close()
test_num = len(test_data)
test_data = test_data[-test_episodes:]
print(len(test_data))
success = 0
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(sess, 'models/policy_supervised_' + relation)
print('Model reloaded')
for episode in xrange(len(test_data)):
print('Test sample %d: %s' % (episode, test_data[episode][:-1]))
env = Env(dataPath, test_data[episode])
sample = test_data[episode].split()
state_idx = [
env.entity2id_[sample[0]], env.entity2id_[sample[1]], 0
]
for t in count():
state_vec = env.idx_state(state_idx)
action_probs = policy_nn.predict(state_vec)
action_chosen = np.random.choice(np.arange(action_space),
p=np.squeeze(action_probs))
reward, new_state, done = env.interact(state_idx,
action_chosen)
if done or t == max_steps_test:
if done:
print('Success')
success += 1
print('Episode ends\n')
break
state_idx = new_state
print('Success persentage:', success / test_episodes)
def REINFORCE(training_pairs, policy_nn, num_episodes):
train = training_pairs
success = 0
# path_found = set()
path_found_entity = []
path_relation_found = []
for i_episode in range(num_episodes):
start = time.time()
print('Episode %d' % i_episode)
print('Training sample: ', train[i_episode][:-1])
env = Env(dataPath, train[i_episode])
sample = train[i_episode].split()
state_idx = [env.entity2id_[sample[0]], env.entity2id_[sample[1]], 0]
episode = []
state_batch_negative = []
action_batch_negative = []
for t in count():
state_vec = env.idx_state(state_idx)
action_probs = policy_nn.predict(state_vec)
action_chosen = np.random.choice(np.arange(action_space),
p=np.squeeze(action_probs))
reward, new_state, done = env.interact(state_idx, action_chosen)
if reward == -1: # the action fails for this step
state_batch_negative.append(state_vec)
action_batch_negative.append(action_chosen)
new_state_vec = env.idx_state(new_state)
episode.append(
Transition(state=state_vec,
action=action_chosen,
next_state=new_state_vec,
reward=reward))
if done or t == max_steps:
break
state_idx = new_state
# Discourage the agent when it choose an invalid step
if len(state_batch_negative) != 0:
print('Penalty to invalid steps:', len(state_batch_negative))
policy_nn.update(np.reshape(state_batch_negative, (-1, state_dim)),
-0.05, action_batch_negative)
print('----- FINAL PATH -----')
print('\t'.join(env.path))
print('PATH LENGTH', len(env.path))
print('----- FINAL PATH -----')
# If the agent success, do one optimization
if done == 1:
print('Success')
path_found_entity.append(path_clean(' -> '.join(env.path)))
success += 1
path_length = len(env.path)
length_reward = 1 / path_length
global_reward = 1
# if len(path_found) != 0:
# path_found_embedding = [env.path_embedding(path.split(' -> ')) for path in path_found]
# curr_path_embedding = env.path_embedding(env.path_relations)
# path_found_embedding = np.reshape(path_found_embedding, (-1,embedding_dim))
# cos_sim = cosine_similarity(path_found_embedding, curr_path_embedding)
# diverse_reward = -np.mean(cos_sim)
# print 'diverse_reward', diverse_reward
# total_reward = 0.1*global_reward + 0.8*length_reward + 0.1*diverse_reward
# else:
# total_reward = 0.1*global_reward + 0.9*length_reward
# path_found.add(' -> '.join(env.path_relations))
total_reward = 0.1 * global_reward + 0.9 * length_reward
state_batch = []
action_batch = []
for t, transition in enumerate(episode):
if transition.reward == 0:
state_batch.append(transition.state)
action_batch.append(transition.action)
policy_nn.update(np.reshape(state_batch, (-1, state_dim)),
total_reward, action_batch)
else:
global_reward = -0.05
# length_reward = 1/len(env.path)
state_batch = []
action_batch = []
total_reward = global_reward
for t, transition in enumerate(episode):
if transition.reward == 0:
state_batch.append(transition.state)
action_batch.append(transition.action)
policy_nn.update(np.reshape(state_batch, (-1, state_dim)),
total_reward, action_batch)
print('Failed, Do one teacher guideline')
try:
good_episodes = teacher(sample[0], sample[1], 1, env,
graphpath)
for item in good_episodes:
teacher_state_batch = []
teacher_action_batch = []
total_reward = 0.0 * 1 + 1 * 1 / len(item)
for t, transition in enumerate(item):
teacher_state_batch.append(transition.state)
teacher_action_batch.append(transition.action)
policy_nn.update(np.squeeze(teacher_state_batch), 1,
teacher_action_batch)
except Exception as e:
print('Teacher guideline failed')
print('Episode time: ', time.time() - start)
print('\n')
print('Success percentage:', success / num_episodes)
for path in path_found_entity:
rel_ent = path.split(' -> ')
path_relation = []
for idx, item in enumerate(rel_ent):
if idx % 2 == 0:
path_relation.append(item)
path_relation_found.append(' -> '.join(path_relation))
relation_path_stats = collections.Counter(path_relation_found).items()
relation_path_stats = sorted(relation_path_stats,
key=lambda x: x[1],
reverse=True)
f = open(dataPath + 'tasks/' + relation + '/' + 'path_stats.txt', 'w')
for item in relation_path_stats:
f.write(item[0] + '\t' + str(item[1]) + '\n')
f.close()
print('Path stats saved')
return