def flush_pool(self):
if self.parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=self.master_experience_replay_size)
else:
self.experience_replay_pool = deque(
maxlen=self.master_experience_replay_size)
def __init__(self,
action_set,
slot_set,
disease_symptom,
parameter,
disease_as_action=True):
self.parameter = parameter
symptom_set = set()
for key, v in disease_symptom.items():
# print(key, len(v['symptom'].keys()))
symptom_set = symptom_set | set(list(v['symptom'].keys()))
# exit(0)
self.action_set = action_set
self.slot_set = slot_set
# self.disease_symptom = disease_symptom
if parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=parameter.get("experience_replay_pool_size"))
else:
self.experience_replay_pool = deque(
maxlen=parameter.get("experience_replay_pool_size"))
self.parameter = parameter
self.candidate_disease_list = []
self.candidate_symptom_list = []
#disease_as_action = self.parameter.get("disease_as_action")
#self.action_space = self._build_action_space(disease_symptom,disease_as_action)
self.disease_symptom = self.disease_symptom_clip(
disease_symptom, 2.5, parameter)
self.agent_action = {
"turn": 1,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
def __init__(self, action_set, slot_set, disease_symptom, parameter):
self.parameter = parameter
self.action_set = action_set
self.slot_set = slot_set
self.slot_set.pop("disease")
self.disease_symptom = disease_symptom
self.master_experience_replay_size = 10000
if parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=self.master_experience_replay_size)
else:
self.experience_replay_pool = deque(
maxlen=self.master_experience_replay_size)
self.input_size_dqn_all = {
1: 374,
4: 494,
5: 389,
6: 339,
7: 279,
12: 304,
13: 359,
14: 394,
19: 414
}
self.id2disease = {}
self.id2lowerAgent = {}
self.master_action_space = []
temp_parameter = {}
for key, value in self.input_size_dqn_all.items():
label = str(key)
#print(label)
self.master_action_space.append(label)
#assert len(label) == 1
#label = label[0]
label_all_path = self.parameter.get("file_all")
label_new_path = os.path.join(label_all_path, 'label' + str(label))
disease_symptom = pickle.load(
open(os.path.join(label_new_path, 'disease_symptom.p'), 'rb'))
slot_set = pickle.load(
open(os.path.join(label_new_path, 'slot_set.p'), 'rb'))
action_set = pickle.load(
open(os.path.join(label_new_path, 'action_set.p'), 'rb'))
temp_parameter[label] = copy.deepcopy(parameter)
#print(parameter["saved_model"])
#if parameter.get("train_mode"):
# temp_parameter["saved_model"] = parameter["saved_model"].split('model_d10_agent')[0] + 'lower/' + str(
# label) + '/model_d10_agent' + parameter["saved_model"].split('model_d10_agent')[1]
#else:
#temp_parameter["saved_model"] = parameter["saved_model"].split('model_d10agent')[0] + 'lower/' + str(
# label) + '/model_d10agent' + parameter["saved_model"].split('model_d10agent')[1]
path_list = parameter['saved_model'].split('/')
path_list.insert(-1, 'lower')
path_list.insert(-1, str(label))
temp_parameter[label]['saved_model'] = '/'.join(path_list)
temp_parameter[label]['gamma'] = temp_parameter[label][
'gamma_worker'] # discount factor for the lower agent.
temp_parameter[label]["input_size_dqn"] = value
self.id2lowerAgent[label] = LowerAgent(
action_set=action_set,
slot_set=slot_set,
disease_symptom=disease_symptom,
parameter=temp_parameter[label],
disease_as_action=False)
#model_path = os.path.join(self.parameter.get("label_all_model_path"), label)
# Master policy.
if parameter.get("state_reduced"):
input_size = len(self.slot_set) * 3
else:
input_size = parameter.get("input_size_dqn")
hidden_size = parameter.get("hidden_size_dqn", 300)
output_size = len(self.id2lowerAgent)
if self.parameter.get("disease_as_action") == False:
output_size = len(self.id2lowerAgent) + 1
#print("input_size",input_size)
self.master = DQN2(input_size=input_size,
hidden_size=hidden_size,
output_size=output_size,
parameter=parameter,
named_tuple=('state', 'agent_action', 'reward',
'next_state', 'episode_over'))
self.parameter = parameter
#self.experience_replay_pool = deque(maxlen=parameter.get("experience_replay_pool_size"))
self.current_lower_agent_id = -1
self.behave_prob = 1
print("master:", self.master_action_space)
self.count = 0
self.past_lower_agent_pool = {
key: 0
for key in self.id2lowerAgent.keys()
}
if parameter.get("train_mode") is False:
print("########## master model is restore now ##########")
self.master.restore_model(parameter.get("saved_model"))
self.master.current_net.eval()
self.master.target_net.eval()
for label, agent in self.id2lowerAgent.items():
#print(temp_parameter[label])
self.id2lowerAgent[label].dqn.restore_model(
temp_parameter[label]['saved_model'])
self.id2lowerAgent[label].dqn.current_net.eval()
self.id2lowerAgent[label].dqn.target_net.eval()
self.agent_action = {
"turn": 1,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
class AgentHRL_joint(object):
def __init__(self, action_set, slot_set, disease_symptom, parameter):
self.parameter = parameter
self.action_set = action_set
self.slot_set = slot_set
self.slot_set.pop("disease")
self.disease_symptom = disease_symptom
self.master_experience_replay_size = 10000
if parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=self.master_experience_replay_size)
else:
self.experience_replay_pool = deque(
maxlen=self.master_experience_replay_size)
self.input_size_dqn_all = {
1: 374,
4: 494,
5: 389,
6: 339,
7: 279,
12: 304,
13: 359,
14: 394,
19: 414
}
self.id2disease = {}
self.id2lowerAgent = {}
self.master_action_space = []
temp_parameter = {}
for key, value in self.input_size_dqn_all.items():
label = str(key)
#print(label)
self.master_action_space.append(label)
#assert len(label) == 1
#label = label[0]
label_all_path = self.parameter.get("file_all")
label_new_path = os.path.join(label_all_path, 'label' + str(label))
disease_symptom = pickle.load(
open(os.path.join(label_new_path, 'disease_symptom.p'), 'rb'))
slot_set = pickle.load(
open(os.path.join(label_new_path, 'slot_set.p'), 'rb'))
action_set = pickle.load(
open(os.path.join(label_new_path, 'action_set.p'), 'rb'))
temp_parameter[label] = copy.deepcopy(parameter)
#print(parameter["saved_model"])
#if parameter.get("train_mode"):
# temp_parameter["saved_model"] = parameter["saved_model"].split('model_d10_agent')[0] + 'lower/' + str(
# label) + '/model_d10_agent' + parameter["saved_model"].split('model_d10_agent')[1]
#else:
#temp_parameter["saved_model"] = parameter["saved_model"].split('model_d10agent')[0] + 'lower/' + str(
# label) + '/model_d10agent' + parameter["saved_model"].split('model_d10agent')[1]
path_list = parameter['saved_model'].split('/')
path_list.insert(-1, 'lower')
path_list.insert(-1, str(label))
temp_parameter[label]['saved_model'] = '/'.join(path_list)
temp_parameter[label]['gamma'] = temp_parameter[label][
'gamma_worker'] # discount factor for the lower agent.
temp_parameter[label]["input_size_dqn"] = value
self.id2lowerAgent[label] = LowerAgent(
action_set=action_set,
slot_set=slot_set,
disease_symptom=disease_symptom,
parameter=temp_parameter[label],
disease_as_action=False)
#model_path = os.path.join(self.parameter.get("label_all_model_path"), label)
# Master policy.
if parameter.get("state_reduced"):
input_size = len(self.slot_set) * 3
else:
input_size = parameter.get("input_size_dqn")
hidden_size = parameter.get("hidden_size_dqn", 300)
output_size = len(self.id2lowerAgent)
if self.parameter.get("disease_as_action") == False:
output_size = len(self.id2lowerAgent) + 1
#print("input_size",input_size)
self.master = DQN2(input_size=input_size,
hidden_size=hidden_size,
output_size=output_size,
parameter=parameter,
named_tuple=('state', 'agent_action', 'reward',
'next_state', 'episode_over'))
self.parameter = parameter
#self.experience_replay_pool = deque(maxlen=parameter.get("experience_replay_pool_size"))
self.current_lower_agent_id = -1
self.behave_prob = 1
print("master:", self.master_action_space)
self.count = 0
self.past_lower_agent_pool = {
key: 0
for key in self.id2lowerAgent.keys()
}
if parameter.get("train_mode") is False:
print("########## master model is restore now ##########")
self.master.restore_model(parameter.get("saved_model"))
self.master.current_net.eval()
self.master.target_net.eval()
for label, agent in self.id2lowerAgent.items():
#print(temp_parameter[label])
self.id2lowerAgent[label].dqn.restore_model(
temp_parameter[label]['saved_model'])
self.id2lowerAgent[label].dqn.current_net.eval()
self.id2lowerAgent[label].dqn.target_net.eval()
self.agent_action = {
"turn": 1,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
def initialize(self):
"""
Initializing an dialogue session.
:return: nothing to return.
"""
self.candidate_disease_list = []
self.candidate_symptom_list = []
self.agent_action = {
"turn": None,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
def next(self, state, turn, greedy_strategy, **kwargs):
"""
Taking action based on different methods, e.g., DQN-based AgentDQN, rule-based AgentRule.
Detail codes will be implemented in different sub-class of this class.
:param state: a vector, the representation of current dialogue state.
:param turn: int, the time step of current dialogue session.
:return: the agent action, a tuple consists of the selected agent action and action index.
"""
# disease_symptom are not used in state_rep.
epsilon = self.parameter.get("epsilon")
#print(state["turn"])
if self.parameter.get("state_reduced"):
state_rep = reduced_state_to_representation_last(
state=state, slot_set=self.slot_set,
parameter=self.parameter) # sequence representation.
else:
state_rep = state_to_representation_last(
state=state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"]
) # sequence representation.
#print(len(state_rep))
# Master agent takes an action.
if self.parameter.get("initial_symptom") and state["turn"] > 0:
pass
else:
#print("####")
if greedy_strategy == True:
greedy = random.random()
if greedy < epsilon:
self.master_action_index = random.randint(
0, len(self.id2lowerAgent))
#print(self.master_action_index)
#master_action_index = random.sample(list(self.id2lowerAgent.keys()),1)[0]
else:
self.master_action_index = self.master.predict(
Xs=[state_rep])[1]
# Evaluating mode.
else:
self.master_action_index = self.master.predict(
Xs=[state_rep])[1]
self.behave_prob = 1 - epsilon + epsilon / (
len(self.id2lowerAgent) - 1)
#print(master_action_index)
if self.parameter.get("prioritized_replay"):
# print('2')
Ys = self.master.predict(Xs=[state_rep])[0]
self.current_action_value = Ys.detach().cpu().numpy()[0][
self.master_action_index]
#print(self.master_action_index) 这里还存在9
# Lower agent takes an agent.
#symptom_dist = self.disease_to_symptom_dist[self.id2disease[self.current_lower_agent_id]]
# 在state_to_representation_last的步骤中，可以自动将不属于slot set中的slot去除掉
if self.parameter.get("disease_as_action"):
self.current_lower_agent_id = self.master_action_space[
self.master_action_index]
agent_action, lower_action_index = self.id2lowerAgent[str(
self.current_lower_agent_id)].next(
state, turn, greedy_strategy=greedy_strategy)
else:
if self.master_action_index > (len(self.id2lowerAgent) - 1):
agent_action = {
'action': 'inform',
'inform_slots': {
"disease": 'UNK'
},
'request_slots': {},
"explicit_inform_slots": {},
"implicit_inform_slots": {}
}
agent_action["turn"] = turn
agent_action["inform_slots"] = {"disease": None}
agent_action["speaker"] = 'agent'
agent_action["action_index"] = None
lower_action_index = -1
else:
self.current_lower_agent_id = self.master_action_space[
self.master_action_index]
#print(self.current_lower_agent_id)
agent_action, lower_action_index = self.id2lowerAgent[str(
self.current_lower_agent_id)].next(
state, turn, greedy_strategy=greedy_strategy)
assert len(list(agent_action["request_slots"].keys())) == 1
#print(self.current_lower_agent_id, lower_action_index)
#print(agent_action)
return agent_action, self.master_action_index, lower_action_index
def next_state_values_DDQN(self, next_state):
if self.parameter.get("state_reduced"):
state_rep = reduced_state_to_representation_last(
state=next_state,
slot_set=self.slot_set,
parameter=self.parameter) # sequence representation.
else:
state_rep = state_to_representation_last(
state=next_state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
action_index = self.master.predict(Xs=[state_rep])[1]
Ys = self.master.predict_target(Xs=[state_rep])
next_action_value = Ys.detach().cpu().numpy()[0][action_index]
return next_action_value
def train(self, batch):
"""
Training the agent.
Args:
batch: the sample used to training.
Return:
dict with a key `loss` whose value it a float.
"""
loss = self.master.singleBatch(
batch=batch,
params=self.parameter,
weight_correction=self.parameter.get("weight_correction"))
return loss
def update_target_network(self):
self.master.update_target_network()
for key in self.id2lowerAgent.keys():
self.id2lowerAgent[key].update_target_network()
def save_model(self,
model_performance,
episodes_index,
checkpoint_path=None):
# Saving master agent
self.master.save_model(model_performance=model_performance,
episodes_index=episodes_index,
checkpoint_path=checkpoint_path)
#Saving lower agent
for key, lower_agent in self.id2lowerAgent.items():
temp_checkpoint_path = os.path.join(checkpoint_path,
'lower/' + str(key))
lower_agent.dqn.save_model(model_performance=model_performance,
episodes_index=episodes_index,
checkpoint_path=temp_checkpoint_path)
def train_dqn(self):
"""
Train dqn.
:return:
"""
# ('state', 'agent_action', 'reward', 'next_state', 'episode_over')
# Training of master agent
cur_bellman_err = 0.0
batch_size = self.parameter.get("batch_size", 16)
#print(batch_size)
priority_scale = self.parameter.get("priority_scale")
if self.parameter.get("prioritized_replay"):
for iter in range(
math.ceil(self.experience_replay_pool.__len__() /
batch_size)):
batch = self.experience_replay_pool.sample(
batch_size=min(batch_size,
self.experience_replay_pool.__len__()),
priority_scale=priority_scale)
loss = self.train(batch=batch)
cur_bellman_err += loss["loss"]
print(
"[Master agent] cur bellman err %.4f, experience replay pool %s"
% (float(cur_bellman_err) /
(self.experience_replay_pool.__len__() + 1e-10),
self.experience_replay_pool.__len__()))
for disease_id, lower_agent in self.id2lowerAgent.items():
if len(lower_agent.experience_replay_pool) > 120:
lower_agent.train_dqn()
else:
for iter in range(
math.ceil(len(self.experience_replay_pool) / batch_size)):
batch = random.sample(
self.experience_replay_pool,
min(batch_size, len(self.experience_replay_pool)))
loss = self.train(batch=batch)
cur_bellman_err += loss["loss"]
print(
"[Master agent] cur bellman err %.4f, experience replay pool %s"
% (float(cur_bellman_err) /
(len(self.experience_replay_pool) + 1e-10),
len(self.experience_replay_pool)))
if self.count % 10 == 9:
for group_id, lower_agent in self.id2lowerAgent.items():
#if len(lower_agent.experience_replay_pool) ==10000 or (len(lower_agent.experience_replay_pool)-self.past_lower_agent_pool[group_id])>100:
if len(lower_agent.experience_replay_pool) > 100:
lower_agent.train_dqn(label=group_id)
self.past_lower_agent_pool[group_id] = len(
lower_agent.experience_replay_pool)
self.count += 1
# Training of lower agents.
#for disease_id, lower_agent in self.id2lowerAgent.items():
# lower_agent.train_dqn()
def reward_shaping(self, state, next_state):
def delete_item_from_dict(item, value):
new_item = {}
for k, v in item.items():
if v != value: new_item[k] = v
return new_item
# slot number in state.
slot_dict = copy.deepcopy(state["current_slots"]["inform_slots"])
slot_dict.update(state["current_slots"]["explicit_inform_slots"])
slot_dict.update(state["current_slots"]["implicit_inform_slots"])
slot_dict.update(state["current_slots"]["proposed_slots"])
slot_dict.update(state["current_slots"]["agent_request_slots"])
slot_dict = delete_item_from_dict(slot_dict,
dialogue_configuration.I_DO_NOT_KNOW)
next_slot_dict = copy.deepcopy(
next_state["current_slots"]["inform_slots"])
next_slot_dict.update(
next_state["current_slots"]["explicit_inform_slots"])
next_slot_dict.update(
next_state["current_slots"]["implicit_inform_slots"])
next_slot_dict.update(next_state["current_slots"]["proposed_slots"])
next_slot_dict.update(
next_state["current_slots"]["agent_request_slots"])
next_slot_dict = delete_item_from_dict(
next_slot_dict, dialogue_configuration.I_DO_NOT_KNOW)
gamma = self.parameter.get("gamma")
return gamma * len(next_slot_dict) - len(slot_dict)
def record_training_sample(self, state, agent_action, reward, next_state,
episode_over, lower_reward):
# samples of master agent.
#print(state)
#print(reward)
shaping = self.reward_shaping(state, next_state)
alpha = self.parameter.get("weight_for_reward_shaping")
'''
if reward == self.parameter.get("reward_for_repeated_action"):
lower_reward = reward
#reward = reward * 2
else:
lower_reward = max(0, shaping * alpha)
#lower_reward = shaping * alpha
'''
if episode_over is True:
pass
else:
reward = reward + alpha * shaping
# samples of lower agent.
#print(agent_action)
if int(agent_action) >= 0:
#print(lower_reward)
self.id2lowerAgent[
self.current_lower_agent_id].record_training_sample(
state, agent_action, lower_reward, next_state,
episode_over)
if self.parameter.get("state_reduced"):
state_rep = reduced_state_to_representation_last(
state=state, slot_set=self.slot_set,
parameter=self.parameter) # sequence representation.
next_state_rep = reduced_state_to_representation_last(
state=next_state,
slot_set=self.slot_set,
parameter=self.parameter)
else:
state_rep = state_to_representation_last(
state=state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
next_state_rep = state_to_representation_last(
state=next_state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
#print("state", [idx for idx,x in enumerate(state_rep) if x==1], agent_action)
#print("nexts", [idx for idx,x in enumerate(next_state_rep) if x==1], reward)
if self.parameter.get("value_as_reward") is True:
q_values = self.id2lowerAgent[
self.current_lower_agent_id].get_q_values(state)
q_values.reshape(q_values.shape[1])
master_reward = np.max(q_values, axis=1)[0]
else:
master_reward = reward
#print(master_reward)
self.experience_replay_pool.append(
(state_rep, self.master_action_index, master_reward,
next_state_rep, episode_over))
def record_prioritized_training_sample(self, state, agent_action, reward,
next_state, episode_over, TD_error,
**kwargs):
shaping = self.reward_shaping(state, next_state)
alpha = self.parameter.get("weight_for_reward_shaping")
# if True:
# print('shaping', shaping)
# Reward shaping only when non-terminal state.
if episode_over is True:
pass
else:
reward = reward + alpha * shaping
if self.parameter.get("state_reduced"):
state_rep = reduced_state_to_representation_last(
state=state, slot_set=self.slot_set,
parameter=self.parameter) # sequence representation.
next_state_rep = reduced_state_to_representation_last(
state=next_state,
slot_set=self.slot_set,
parameter=self.parameter)
else:
state_rep = state_to_representation_last(
state=state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
next_state_rep = state_to_representation_last(
state=next_state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
self.experience_replay_pool.add(state_rep, agent_action, reward,
next_state_rep, episode_over, TD_error)
def flush_pool(self):
if self.parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=self.master_experience_replay_size)
else:
self.experience_replay_pool = deque(
maxlen=self.master_experience_replay_size)
#for key, lower_agent in self.id2lowerAgent.items():
# self.id2lowerAgent[key].flush_pool()
def train_mode(self):
self.master.current_net.train()
def eval_mode(self):
self.master.current_net.eval()
class Agent(object):
"""
Basic class of agent.
"""
def __init__(self,
action_set,
slot_set,
disease_symptom,
parameter,
disease_as_action=True):
self.parameter = parameter
symptom_set = set()
for key, v in disease_symptom.items():
# print(key, len(v['symptom'].keys()))
symptom_set = symptom_set | set(list(v['symptom'].keys()))
# exit(0)
self.action_set = action_set
self.slot_set = slot_set
# self.disease_symptom = disease_symptom
if parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=parameter.get("experience_replay_pool_size"))
else:
self.experience_replay_pool = deque(
maxlen=parameter.get("experience_replay_pool_size"))
self.parameter = parameter
self.candidate_disease_list = []
self.candidate_symptom_list = []
#disease_as_action = self.parameter.get("disease_as_action")
#self.action_space = self._build_action_space(disease_symptom,disease_as_action)
self.disease_symptom = self.disease_symptom_clip(
disease_symptom, 2.5, parameter)
self.agent_action = {
"turn": 1,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
def initialize(self):
"""
Initializing an dialogue session.
:return: nothing to return.
"""
self.candidate_disease_list = []
self.candidate_symptom_list = []
self.agent_action = {
"turn": None,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
def next(self, *args, **kwargs):
"""
Taking action based on different methods, e.g., DQN-based AgentDQN, rule-based AgentRule.
Detail codes will be implemented in different sub-class of this class.
:param state: a vector, the representation of current dialogue state.
:param turn: int, the time step of current dialogue session.
:return: the agent action, a tuple consists of the selected agent action and action index.
"""
raise NotImplementedError(
'The `next` function of agent has not been implemented.')
def train(self, batch):
"""
Training the agent.
Detail codes will be implemented in different sub-class of this class.
:param batch: the sample used to training.
:return:
"""
raise NotImplementedError(
'The `train` function of agent has not been implemented.')
def _build_action_space(self, disease_symptom, disease_as_action):
"""
Building the Action Space for the RL-based Agent.
All diseases are treated as actions.
:return: Action Space, a list of feasible actions.
"""
feasible_actions = []
# Adding the request actions. And the slots are extracted from the links between disease and symptom,
# i.e., disease_symptom
slot_set = []
for disease, v in disease_symptom.items():
slot_set = slot_set + list(v["symptom"])
slot_set = list(set(slot_set))
for slot in sorted(slot_set):
if slot != "disease":
feasible_actions.append({
'action': 'request',
'inform_slots': {},
'request_slots': {
slot: dialogue_configuration.VALUE_UNKNOWN
},
"explicit_inform_slots": {},
"implicit_inform_slots": {}
})
# Diseases as actions: inform + disease.
if self.parameter.get("agent_id").lower() == "agenthrljoint":
if disease_as_action is True:
for disease in sorted(disease_symptom.keys()):
feasible_actions.append({
'action': 'inform',
'inform_slots': {
"disease": disease
},
'request_slots': {},
"explicit_inform_slots": {},
"implicit_inform_slots": {}
})
elif self.parameter.get("agent_id").lower() == "agenthrljoint2":
if disease_as_action is True:
for disease in sorted(disease_symptom.keys()):
feasible_actions.append({
'action': 'inform',
'inform_slots': {
"disease": disease
},
'request_slots': {},
"explicit_inform_slots": {},
"implicit_inform_slots": {}
})
#else:
# feasible_actions.append({'action': "return", 'inform_slots': {}, 'request_slots': {},"explicit_inform_slots":{}, "implicit_inform_slots":{}})
else:
#print("#########################")
if disease_as_action is True:
for disease in sorted(disease_symptom.keys()):
feasible_actions.append({
'action': 'inform',
'inform_slots': {
"disease": disease
},
'request_slots': {},
"explicit_inform_slots": {},
"implicit_inform_slots": {}
})
else:
feasible_actions.append({
'action': "inform",
'inform_slots': {
"disease": None
},
'request_slots': {},
"explicit_inform_slots": {},
"implicit_inform_slots": {}
})
# greeting actions includes Thanks and close dialogue.
# feasible_actions.append({'action': "confirm_question", 'inform_slots': {}, 'request_slots': {},"explicit_inform_slots":{}, "implicit_inform_slots":{}})
# feasible_actions.append({'action': "confirm_answer", 'inform_slots': {}, 'request_slots': {},"explicit_inform_slots":{}, "implicit_inform_slots":{}})
# feasible_actions.append({'action': "deny", 'inform_slots': {}, 'request_slots': {},"explicit_inform_slots":{}, "implicit_inform_slots":{}})
# feasible_actions.append({'action': dialogue_configuration.CLOSE_DIALOGUE, 'inform_slots': {}, 'request_slots': {},"explicit_inform_slots":{}, "implicit_inform_slots":{}})
# feasible_actions.append({'action': dialogue_configuration.THANKS, 'inform_slots': {}, 'request_slots': {}, "explicit_inform_slots": {}, "implicit_inform_slots": {}})
return feasible_actions
@staticmethod
def disease_symptom_clip(disease_symptom, denominator, parameter):
"""
Keep the top min(symptom_num, max_turn//denominator) for each disease, and the related symptoms are sorted
descendent according to their frequencies.
Args:
disease_symptom: a dict, key is the names of diseases, and the corresponding value is a dict too which
contains the index of this disease and the related symptoms.
denominator: int, the number of symptoms for each diseases is max_turn // denominator.
parameter: the super-parameter.
Returns:
and dict, whose keys are the names of diseases, and the values are dicts too with two keys: {'index', symptom}
"""
max_turn = parameter.get('max_turn')
temp_disease_symptom = copy.deepcopy(disease_symptom)
for key, value in disease_symptom.items():
symptom_list = sorted(value['symptom'].items(),
key=lambda x: x[1],
reverse=True)
symptom_list = [v[0] for v in symptom_list]
symptom_list = symptom_list[
0:min(len(symptom_list), int(max_turn / float(denominator)))]
temp_disease_symptom[key]['symptom'] = symptom_list
return temp_disease_symptom
def record_training_sample(self, state, agent_action, reward, next_state,
episode_over, **kwargs):
symptom_dist_as_input = self.parameter.get("symptom_dist_as_input")
agent_id = self.parameter.get("agent_id")
if self.parameter.get("state_reduced"):
state = reduced_state_to_representation_last(
state=state,
slot_set=self.slot_set) # sequence representation.
next_state = reduced_state_to_representation_last(
state=next_state, slot_set=self.slot_set)
else:
state = state_to_representation_last(
state=state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
next_state = state_to_representation_last(
state=next_state,
action_set=self.action_set,
slot_set=self.slot_set,
disease_symptom=self.disease_symptom,
max_turn=self.parameter["max_turn"])
if symptom_dist_as_input is True and agent_id.lower() == 'agenthrl':
symptom_dist = kwargs.get('symptom_dist')
state = np.concatenate((state, symptom_dist), axis=0)
next_state = np.concatenate((next_state, symptom_dist), axis=0)
self.experience_replay_pool.append(
(state, agent_action, reward, next_state, episode_over))
def flush_pool(self):
if self.parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=self.parameter.get("experience_replay_pool_size"))
else:
self.experience_replay_pool = deque(
maxlen=self.parameter.get("experience_replay_pool_size"))
def train_mode(self):
"""
Set the agent as the train mode, i.e., the parameters will be updated and dropout will be activated.
"""
raise NotImplementedError(
"The `train_mode` function of agent has not been implemented")
def eval_mode(self):
"""
Set the agent as the train mode, i.e., the parameters will be unchanged and dropout will be deactivated.
"""
raise NotImplementedError(
"The `train_mode` function of agent has not been implemented")
def __init__(self, action_set, slot_set, disease_symptom, parameter):
self.parameter = parameter
self.action_set = action_set
self.slot_set = slot_set
#self.slot_set.pop("disease")
self.disease_symptom = disease_symptom
self.master_experience_replay_size = 10000
if parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=self.master_experience_replay_size)
else:
self.experience_replay_pool = deque(
maxlen=self.master_experience_replay_size)
if self.parameter.get("data_type") == 'simulated':
self.input_size_dqn_all = {
1: 374,
4: 494,
5: 389,
6: 339,
7: 279,
12: 304,
13: 359,
14: 394,
19: 414
}
elif self.parameter.get("data_type") == 'real':
self.input_size_dqn_all = {0: 84, 1: 81, 2: 81, 3: 83}
else:
raise ValueError
self.id2disease = {}
self.id2lowerAgent = {}
self.pretrained_lowerAgent = {}
self.master_action_space = []
temp_parameter = {}
# different label = different disease sysptom
for key, value in self.input_size_dqn_all.items():
#dirs = os.listdir(self.parameter.get("label_all_model_path"))
#for model in dirs:
#reg = re.compile(r"(?<=label)\d+")
#match = reg.search(model)
#label = match.group(0)
# print(label)
label = str(key)
self.master_action_space.append(label)
# assert len(label) == 1
# label = label[0]
label_all_path = self.parameter.get("file_all")
label_new_path = os.path.join(label_all_path, 'label' + str(label))
disease_symptom = pickle.load(
open(os.path.join(label_new_path, 'disease_symptom.p'), 'rb'))
slot_set = pickle.load(
open(os.path.join(label_new_path, 'slot_set.p'), 'rb'))
action_set = pickle.load(
open(os.path.join(label_new_path, 'action_set.p'), 'rb'))
temp_parameter[label] = copy.deepcopy(parameter)
# print(parameter["saved_model"])
# if parameter.get("train_mode"):
# temp_parameter["saved_model"] = parameter["saved_model"].split('model_d10_agent')[0] + 'lower/' + str(
# label) + '/model_d10_agent' + parameter["saved_model"].split('model_d10_agent')[1]
# else:
path_list = parameter['saved_model'].split('/')
path_list.insert(-1, 'lower')
path_list.insert(-1, str(label))
temp_parameter[label]['saved_model'] = '/'.join(path_list)
temp_parameter[label]['gamma'] = temp_parameter[label][
'gamma_worker'] # discount factor for the lower agent.
temp_parameter[label]["input_size_dqn"] = self.input_size_dqn_all[
int(label)]
#temp_parameter[label]["input_size_dqn"] = (len(slot_set)-1) *3
self.id2lowerAgent[label] = LowerAgent(
action_set=action_set,
slot_set=slot_set,
disease_symptom=disease_symptom,
parameter=temp_parameter[label],
disease_as_action=False)
# model_path = os.path.join(self.parameter.get("label_all_model_path"), label)
#temp_parameter[label]["input_size_dqn"] = self.input_size_dqn_all[int(label)]
'''
temp_parameter[label]["input_size_dqn"] = (len(slot_set)) * 3
#print(slot_set)
self.pretrained_lowerAgent[label] = LowerAgent(action_set=action_set, slot_set=slot_set,
disease_symptom=disease_symptom, parameter=temp_parameter[label],
disease_as_action=True)
# model_path = os.path.join(self.parameter.get("label_all_model_path"), label)
self.pretrained_lowerAgent[label].dqn.restore_model(os.path.join(self.parameter.get("label_all_model_path"), model))
self.pretrained_lowerAgent[label].dqn.current_net.eval()
self.pretrained_lowerAgent[label].dqn.target_net.eval()
'''
# Master policy.
if parameter.get("state_reduced"):
input_size = (
len(self.slot_set) - 1
) * 3 # the dictionary of slot_set contains a key of "disease" which need to be removed first.
else:
input_size = parameter.get("input_size_dqn")
hidden_size = parameter.get("hidden_size_dqn", 300)
# 患病种类的数量
self.output_size = len(self.id2lowerAgent)
if self.parameter.get("disease_as_action") == False:
self.output_size = len(
self.id2lowerAgent
) + 1 # the extra one size is the action of activating disease classifier
#print("input_size",input_size)
self.master = DQN2(input_size=input_size,
hidden_size=hidden_size,
output_size=self.output_size,
parameter=parameter,
named_tuple=('state', 'agent_action', 'reward',
'next_state', 'episode_over'))
self.parameter = parameter
# self.experience_replay_pool = deque(maxlen=parameter.get("experience_replay_pool_size"))
self.current_lower_agent_id = -1
self.behave_prob = 1
print("master:", self.master_action_space)
self.count = 0
self.subtask_terminal = True
self.subtask_turn = 0
self.subtask_max_turn = 5
self.past_lower_agent_pool = {
key: 0
for key in self.id2lowerAgent.keys()
}
if parameter.get("train_mode") is False:
print("########## master model is restore now ##########")
self.master.restore_model(parameter.get("saved_model"))
self.master.current_net.eval()
self.master.target_net.eval()
for label, agent in self.id2lowerAgent.items():
#print(temp_parameter[label])
self.id2lowerAgent[label].dqn.restore_model(
temp_parameter[label]['saved_model'])
self.id2lowerAgent[label].dqn.current_net.eval()
self.id2lowerAgent[label].dqn.target_net.eval()
self.agent_action = {
"turn": 1,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}
def __init__(self, action_set, slot_set, disease_symptom, parameter):
self.parameter = parameter
self.action_set = action_set
self.slot_set = slot_set
self.slot_set.pop("disease")
self.disease_symptom = disease_symptom
if parameter.get('prioritized_replay'):
self.experience_replay_pool = PrioritizedReplayBuffer(
buffer_size=parameter.get("experience_replay_pool_size"))
else:
self.experience_replay_pool = deque(
maxlen=parameter.get("experience_replay_pool_size"))
self.input_size_dqn_all = {
1: 374,
4: 494,
5: 389,
6: 339,
7: 279,
9: 409,
10: 254,
11: 304,
12: 304,
13: 359,
14: 394,
19: 414
}
self.id2disease = {}
self.id2lowerAgent = {}
self.master_action_space = []
dirs = os.listdir(self.parameter.get("label_all_model_path"))
for model in dirs:
#pattern = re.compile(r'(?<=label=)\d+\.?\d*')
#label = pattern.findall(model)
reg = re.compile(r"(?<=label)\d+")
match = reg.search(model)
label = match.group(0)
#print(label)
self.master_action_space.append(label)
#assert len(label) == 1
#label = label[0]
label_all_path = self.parameter.get("file_all")
label_new_path = os.path.join(label_all_path, 'label' + label)
disease_symptom = pickle.load(
open(os.path.join(label_new_path, 'disease_symptom.p'), 'rb'))
slot_set = pickle.load(
open(os.path.join(label_new_path, 'slot_set.p'), 'rb'))
action_set = pickle.load(
open(os.path.join(label_new_path, 'action_set.p'), 'rb'))
temp_parameter = copy.deepcopy(parameter)
#print(parameter["saved_model"])
#if parameter.get("train_mode"):
# temp_parameter["saved_model"] = parameter["saved_model"].split('model_d10_agent')[0] + 'lower/' + str(
# label) + '/model_d10_agent' + parameter["saved_model"].split('model_d10_agent')[1]
#else:
temp_parameter["saved_model"] = parameter["saved_model"].split(
'model_d10agent')[0] + 'lower/' + str(
label) + '/model_d10agent' + parameter[
"saved_model"].split('model_d10agent')[1]
temp_parameter["input_size_dqn"] = self.input_size_dqn_all[int(
label)]
self.id2lowerAgent[label] = LowerAgent(
action_set=action_set,
slot_set=slot_set,
disease_symptom=disease_symptom,
parameter=temp_parameter,
disease_as_action=True)
#model_path = os.path.join(self.parameter.get("label_all_model_path"), label)
self.id2lowerAgent[label].dqn.restore_model(
os.path.join(self.parameter.get("label_all_model_path"),
model))
self.id2lowerAgent[label].dqn.current_net.eval()
self.id2lowerAgent[label].dqn.target_net.eval()
# Master policy.
if parameter.get("state_reduced"):
input_size = len(self.slot_set) * 3
else:
input_size = parameter.get("input_size_dqn")
hidden_size = parameter.get("hidden_size_dqn", 100)
output_size = len(self.id2lowerAgent)
if self.parameter.get("disease_as_action") == False:
output_size = len(self.id2lowerAgent) + 1
#print("input_size",input_size)
self.master = DQN2(input_size=input_size,
hidden_size=hidden_size,
output_size=output_size,
parameter=parameter,
named_tuple=('state', 'agent_action', 'reward',
'next_state', 'episode_over'))
self.parameter = parameter
#self.experience_replay_pool = deque(maxlen=parameter.get("experience_replay_pool_size"))
self.current_lower_agent_id = -1
self.behave_prob = 1
print("master:", self.master_action_space)
if parameter.get("train_mode") is False:
print("########## master model is restore now ##########")
self.master.restore_model(parameter.get("saved_model"))
self.master.current_net.eval()
self.master.target_net.eval()
self.agent_action = {
"turn": 1,
"action": None,
"request_slots": {},
"inform_slots": {},
"explicit_inform_slots": {},
"implicit_inform_slots": {},
"speaker": "agent"
}