def initialize(self, **kwargs):
"""
Initialize internal structures at the beginning of each dialogue
:return: Nothing
"""
if self.agent_role == 'system':
# Put your system expert policy here
self.warmup_policy = HandcraftedPolicy(self.ontology)
elif self.agent_role == 'user':
usim_args = \
dict(
zip(['ontology', 'database'],
[self.ontology, self.database]))
# Put your user expert policy here
self.warmup_simulator = AgendaBasedUS(usim_args)
if 'is_training' in kwargs:
self.is_training = bool(kwargs['is_training'])
if self.agent_role == 'user' and self.warmup_simulator:
if 'goal' in kwargs:
self.warmup_simulator.initialize({kwargs['goal']})
else:
print('WARNING ! No goal provided for Supervised policy '
'user simulator @ initialize')
self.warmup_simulator.initialize({})
if 'policy_path' in kwargs:
self.policy_path = kwargs['policy_path']
if 'learning_rate' in kwargs:
self.policy_alpha = kwargs['learning_rate']
if self.sess is None:
self.policy_net = self.feed_forward_net_init()
self.sess = tf.InteractiveSession()
self.sess.run(tf.global_variables_initializer())
self.tf_saver = \
tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.GLOBAL_VARIABLES, scope=self.tf_scope))
def __init__(self,
ontology,
database,
agent_id=0,
agent_role='system',
alpha=0.25,
gamma=0.95,
epsilon=0.25,
alpha_decay=0.9995,
epsilon_decay=0.995):
"""
Initialize parameters and internal structures
:param ontology: the domain's ontology
:param database: the domain's database
:param agent_id: the agent's id
:param agent_role: the agent's role
:param alpha: the learning rate
:param gamma: the discount rate
:param epsilon: the exploration rate
:param alpha_decay: the learning rate discount rate
:param epsilon_decay: the exploration rate discount rate
"""
self.alpha = alpha
self.gamma = gamma
self.epsilon = epsilon
self.alpha_decay = alpha_decay
self.epsilon_decay = epsilon_decay
self.is_training = False
self.agent_id = agent_id
self.agent_role = agent_role
self.ontology = None
if isinstance(ontology, Ontology):
self.ontology = ontology
else:
raise ValueError('Unacceptable ontology type %s ' % ontology)
self.database = None
if isinstance(database, DataBase):
self.database = database
else:
raise ValueError('MinimaxQ DialoguePolicy: Unacceptable database '
'type %s ' % database)
self.Q = {}
self.V = {}
self.pi = {}
self.pp = pprint.PrettyPrinter(width=160) # For debug!
# System and user expert policies (optional)
self.warmup_policy = None
self.warmup_simulator = None
if self.agent_role == 'system':
# Put your system expert policy here
self.warmup_policy = \
HandcraftedPolicy.HandcraftedPolicy(self.ontology)
elif self.agent_role == 'user':
usim_args = \
dict(
zip(['ontology', 'database'],
[self.ontology, self.database]))
# Put your user expert policy here
self.warmup_simulator = AgendaBasedUS(usim_args)
# Sub-case for CamRest
self.dstc2_acts_sys = self.dstc2_acts_usr = None
# Plato does not use action masks (rules to define which
# actions are valid from each state) and so training can
# be harder. This becomes easier if we have a smaller
# action set.
# Does not include inform and request that are modelled together with
# their arguments
self.dstc2_acts_sys = [
'offer', 'canthelp', 'affirm', 'deny', 'ack', 'bye', 'reqmore',
'welcomemsg', 'expl-conf', 'select', 'repeat', 'confirm-domain',
'confirm'
]
# Does not include inform and request that are modelled together with
# their arguments
self.dstc2_acts_usr = [
'affirm', 'negate', 'deny', 'ack', 'thankyou', 'bye', 'reqmore',
'hello', 'expl-conf', 'repeat', 'reqalts', 'restart', 'confirm'
]
# Extract lists of slots that are frequently used
self.informable_slots = \
deepcopy(list(self.ontology.ontology['informable'].keys()))
self.requestable_slots = \
deepcopy(self.ontology.ontology['requestable'])
self.system_requestable_slots = \
deepcopy(self.ontology.ontology['system_requestable'])
if self.dstc2_acts_sys:
if self.agent_role == 'system':
self.NActions = \
len(self.dstc2_acts_sys) + \
len(self.requestable_slots) + \
len(self.system_requestable_slots)
self.NOtherActions = \
len(self.dstc2_acts_usr) + \
2 * len(self.requestable_slots)
elif self.agent_role == 'user':
self.NActions = \
len(self.dstc2_acts_usr) + \
2 * len(self.requestable_slots)
self.NOtherActions = \
len(self.dstc2_acts_sys) + \
len(self.requestable_slots) + \
len(self.system_requestable_slots)
else:
if self.agent_role == 'system':
self.NActions = \
5 + \
len(self.ontology.ontology['system_requestable']) + \
len(self.ontology.ontology['requestable'])
self.NOtherActions = \
4 + 2 * len(self.ontology.ontology['requestable'])
elif self.agent_role == 'user':
self.NActions = \
4 + 2 * len(self.ontology.ontology['requestable'])
self.NOtherActions = \
5 + len(self.ontology.ontology['system_requestable']) + \
len(self.ontology.ontology['requestable'])
def __init__(self,
ontology,
database,
agent_id=0,
agent_role='system',
domain=None,
alpha=0.2,
epsilon=0.95,
gamma=0.95,
alpha_decay=0.995,
epsilon_decay=0.9995,
epsilon_min=0.05):
"""
Initialize parameters and internal structures
:param ontology: the domain's ontology
:param database: the domain's database
:param agent_id: the agent's id
:param agent_role: the agent's role
:param alpha: the learning rate
:param gamma: the discount rate
:param epsilon: the exploration rate
:param alpha_decay: the learning rate discount rate
:param epsilon_decay: the exploration rate discount rate
"""
super(ReinforcePolicy, self).__init__()
self.logger = logging.getLogger(__name__)
self.agent_id = agent_id
self.agent_role = agent_role
self.IS_GREEDY = False
self.ontology = None
if isinstance(ontology, Ontology.Ontology):
self.ontology = ontology
else:
raise ValueError('Unacceptable ontology type %s ' % ontology)
self.database = None
if isinstance(database, DataBase.DataBase):
self.database = database
else:
raise ValueError('Reinforce DialoguePolicy: Unacceptable '
'database type %s ' % database)
self.policy_path = None
self.weights = None
self.sess = None
self.alpha = alpha
self.gamma = gamma
self.epsilon = epsilon
self.alpha_decay_rate = alpha_decay
self.exploration_decay_rate = epsilon_decay
self.epsilon_min = epsilon_min
# System and user expert policies (optional)
self.warmup_policy = None
self.warmup_simulator = None
if self.agent_role == 'system':
# Put your system expert policy here
self.warmup_policy = HandcraftedPolicy(self.ontology)
elif self.agent_role == 'user':
usim_args = \
dict(
zip(['ontology', 'database'],
[self.ontology, self.database]))
# Put your user expert policy here
self.warmup_simulator = AgendaBasedUS(usim_args)
self.tf_scope = "policy_" + self.agent_role + '_' + str(self.agent_id)
# Default value
self.is_training = True
# Extract lists of slots that are frequently used
self.informable_slots = \
deepcopy(list(self.ontology.ontology['informable'].keys()))
self.requestable_slots = \
deepcopy(self.ontology.ontology['requestable'])
self.system_requestable_slots = \
deepcopy(self.ontology.ontology['system_requestable'])
if not domain:
# Default to CamRest dimensions
self.NStateFeatures = 56
# Default to CamRest actions
self.dstc2_acts = [
'inform', 'offer', 'request', 'canthelp', 'affirm', 'negate',
'deny', 'ack', 'thankyou', 'bye', 'reqmore', 'hello',
'welcomemsg', 'expl-conf', 'select', 'repeat', 'reqalts',
'confirm-domain', 'confirm'
]
else:
# Try to identify number of state features
if domain in ['CamRest', 'SFH', 'SlotFilling']:
d_state = \
SlotFillingDialogueState(
{'slots': self.system_requestable_slots})
# Plato does not use action masks (rules to define which
# actions are valid from each state) and so training can
# be harder. This becomes easier if we have a smaller
# action set.
# Sub-case for CamRest
if domain == 'CamRest':
# Does not include inform and request that are modelled
# together with their arguments
self.dstc2_acts_sys = [
'offer', 'canthelp', 'affirm', 'deny', 'ack', 'bye',
'reqmore', 'welcomemsg', 'expl-conf', 'select',
'repeat', 'confirm-domain', 'confirm'
]
# Does not include inform and request that are modelled
# together with their arguments
self.dstc2_acts_usr = [
'affirm', 'negate', 'deny', 'ack', 'thankyou', 'bye',
'reqmore', 'hello', 'expl-conf', 'repeat', 'reqalts',
'restart', 'confirm'
]
else:
self.logger.warning(
'Warning! Domain has not been defined. Using '
'Slot-Filling Dialogue State')
d_state = \
SlotFillingDialogueState({'slots': self.informable_slots})
d_state.initialize()
self.NStateFeatures = len(self.encode_state(d_state))
self.logger.info(
'Reinforce DialoguePolicy {0} automatically determined '
'number of state features: {1}'.format(self.agent_role,
self.NStateFeatures))
if domain == 'CamRest' and self.dstc2_acts_sys:
if self.agent_role == 'system':
self.NActions = \
len(self.dstc2_acts_sys) + \
len(self.requestable_slots) + \
len(self.system_requestable_slots)
self.NOtherActions = \
len(self.dstc2_acts_usr) + \
2 * len(self.requestable_slots)
elif self.agent_role == 'user':
self.NActions = \
len(self.dstc2_acts_usr) + \
len(self.requestable_slots) + \
len(self.system_requestable_slots)
self.NOtherActions = \
len(self.dstc2_acts_sys) + \
len(self.requestable_slots) + \
len(self.system_requestable_slots)
else:
if self.agent_role == 'system':
self.NActions = \
3 + len(self.system_requestable_slots) + \
len(self.requestable_slots)
self.NOtherActions = \
2 + len(self.requestable_slots) +\
len(self.requestable_slots)
elif self.agent_role == 'user':
self.NActions = \
2 + len(self.requestable_slots) + \
len(self.requestable_slots)
self.NOtherActions = \
3 + len(self.system_requestable_slots) + \
len(self.requestable_slots)
self.logger.info(
'Reinforce {0} DialoguePolicy Number of Actions: {1}'.format(
self.agent_role, self.NActions))
def __init__(self, configuration):
"""
Initialize the internal structures of this agent.
:param configuration: a dictionary representing the configuration file
:param agent_id: an integer, this agent's id
"""
super(ConversationalSingleAgent, self).__init__()
self.configuration = configuration
# There is only one agent in this setting
self.agent_id = 0
# Dialogue statistics
self.dialogue_episode = 0
self.dialogue_turn = 0
self.num_successful_dialogues = 0
self.num_task_success = 0
self.cumulative_rewards = 0
self.total_dialogue_turns = 0
self.minibatch_length = 500
self.train_interval = 50
self.train_epochs = 10
# True values here would imply some default modules
self.USE_USR_SIMULATOR = False
self.USER_SIMULATOR_NLU = False
self.USER_SIMULATOR_NLG = False
self.USE_NLG = False
self.USE_SPEECH = False
self.USER_HAS_INITIATIVE = True
self.SAVE_LOG = True
# The dialogue will terminate after MAX_TURNS (this agent will issue
# a bye() dialogue act.
self.MAX_TURNS = 15
self.dialogue_turn = -1
self.ontology = None
self.database = None
self.domain = None
self.dialogue_manager = None
self.user_model = None
self.user_simulator = None
self.user_simulator_args = {}
self.nlu = None
self.nlg = None
self.agent_role = None
self.agent_goal = None
self.goal_generator = None
self.curr_state = None
self.prev_state = None
self.curr_state = None
self.prev_usr_utterance = None
self.prev_sys_utterance = None
self.prev_action = None
self.prev_reward = None
self.prev_success = None
self.prev_task_success = None
self.user_model = UserModel()
self.recorder = DialogueEpisodeRecorder()
# TODO: Handle this properly - get reward function type from config
self.reward_func = SlotFillingReward()
# self.reward_func = SlotFillingGoalAdvancementReward()
if self.configuration:
# Error checks for options the config must have
if not self.configuration['GENERAL']:
raise ValueError('Cannot run Plato without GENERAL settings!')
elif not self.configuration['GENERAL']['interaction_mode']:
raise ValueError('Cannot run Plato without an '
'interaction mode!')
elif not self.configuration['DIALOGUE']:
raise ValueError('Cannot run Plato without DIALOGUE settings!')
elif not self.configuration['AGENT_0']:
raise ValueError('Cannot run Plato without at least '
'one agent!')
# Dialogue domain self.settings
if 'DIALOGUE' in self.configuration and \
self.configuration['DIALOGUE']:
if 'initiative' in self.configuration['DIALOGUE']:
self.USER_HAS_INITIATIVE = bool(
self.configuration['DIALOGUE']['initiative'] == 'user')
self.user_simulator_args['us_has_initiative'] = \
self.USER_HAS_INITIATIVE
if self.configuration['DIALOGUE']['domain']:
self.domain = self.configuration['DIALOGUE']['domain']
if self.configuration['DIALOGUE']['ontology_path']:
if os.path.isfile(
self.configuration['DIALOGUE']['ontology_path']):
self.ontology = Ontology.Ontology(
self.configuration['DIALOGUE']['ontology_path'])
else:
raise FileNotFoundError(
'Domain file %s not found' %
self.configuration['DIALOGUE']['ontology_path'])
if self.configuration['DIALOGUE']['db_path']:
if os.path.isfile(
self.configuration['DIALOGUE']['db_path']):
if 'db_type' in self.configuration['DIALOGUE']:
if self.configuration['DIALOGUE']['db_type'] == \
'sql':
self.database = DataBase.SQLDataBase(
self.configuration['DIALOGUE']['db_path'])
else:
self.database = DataBase.DataBase(
self.configuration['DIALOGUE']['db_path'])
else:
# Default to SQL
self.database = DataBase.SQLDataBase(
self.configuration['DIALOGUE']['db_path'])
else:
raise FileNotFoundError(
'Database file %s not found' %
self.configuration['DIALOGUE']['db_path'])
if 'goals_path' in self.configuration['DIALOGUE']:
if os.path.isfile(
self.configuration['DIALOGUE']['goals_path']):
self.goals_path = \
self.configuration['DIALOGUE']['goals_path']
else:
raise FileNotFoundError(
'Goals file %s not found' %
self.configuration['DIALOGUE']['goals_path'])
# General settings
if 'GENERAL' in self.configuration and \
self.configuration['GENERAL']:
if 'experience_logs' in self.configuration['GENERAL']:
dialogues_path = None
if 'path' in \
self.configuration['GENERAL']['experience_logs']:
dialogues_path = \
self.configuration['GENERAL'][
'experience_logs']['path']
if 'load' in \
self.configuration['GENERAL']['experience_logs'] \
and bool(
self.configuration['GENERAL'][
'experience_logs']['load']
):
if dialogues_path and os.path.isfile(dialogues_path):
self.recorder.load(dialogues_path)
else:
raise FileNotFoundError(
'Dialogue Log file %s not found (did you '
'provide one?)' % dialogues_path)
if 'save' in \
self.configuration['GENERAL']['experience_logs']:
self.recorder.set_path(dialogues_path)
self.SAVE_LOG = bool(self.configuration['GENERAL']
['experience_logs']['save'])
if self.configuration['GENERAL']['interaction_mode'] == \
'simulation':
self.USE_USR_SIMULATOR = True
elif self.configuration['GENERAL']['interaction_mode'] == \
'speech':
self.USE_SPEECH = True
self.asr = speech_rec.Recognizer()
# Agent Settings
# Usr Simulator
# Check for specific simulator self.settings, otherwise
# default to agenda
if 'USER_SIMULATOR' in self.configuration['AGENT_0']:
# Agent 0 simulator configuration
a0_sim_config = self.configuration['AGENT_0']['USER_SIMULATOR']
if a0_sim_config and a0_sim_config['simulator']:
# Default settings
self.user_simulator_args['ontology'] = self.ontology
self.user_simulator_args['database'] = self.database
self.user_simulator_args['um'] = self.user_model
self.user_simulator_args['patience'] = 5
if a0_sim_config['simulator'] == 'agenda':
if 'patience' in a0_sim_config:
self.user_simulator_args['patience'] = \
int(a0_sim_config['patience'])
if 'pop_distribution' in a0_sim_config:
if isinstance(a0_sim_config['pop_distribution'],
list):
self.user_simulator_args['pop_distribution'] =\
a0_sim_config['pop_distribution']
else:
self.user_simulator_args['pop_distribution'] =\
eval(a0_sim_config['pop_distribution'])
if 'slot_confuse_prob' in a0_sim_config:
self.user_simulator_args['slot_confuse_prob'] = \
float(a0_sim_config['slot_confuse_prob'])
if 'op_confuse_prob' in a0_sim_config:
self.user_simulator_args['op_confuse_prob'] = \
float(a0_sim_config['op_confuse_prob'])
if 'value_confuse_prob' in a0_sim_config:
self.user_simulator_args['value_confuse_prob'] = \
float(a0_sim_config['value_confuse_prob'])
if 'goal_slot_selection_weights' in a0_sim_config:
self.user_simulator_args[
'goal_slot_selection_weights'] = a0_sim_config[
'goal_slot_selection_weights']
if 'nlu' in a0_sim_config:
self.user_simulator_args['nlu'] = \
a0_sim_config['nlu']
if self.user_simulator_args['nlu'] == 'dummy':
self.user_simulator_args['database'] = \
self.database
self.USER_SIMULATOR_NLU = True
if 'nlg' in a0_sim_config:
self.user_simulator_args['nlg'] = \
a0_sim_config['nlg']
if self.user_simulator_args['nlg'] == 'CamRest':
if a0_sim_config:
self.user_simulator_args[
'nlg_model_path'] = a0_sim_config[
'nlg_model_path']
self.USER_SIMULATOR_NLG = True
else:
raise ValueError(
'Usr Simulator NLG: Cannot find '
'model_path in the config.')
elif self.user_simulator_args['nlg'] == 'dummy':
self.USER_SIMULATOR_NLG = True
if 'goals_file' in a0_sim_config:
self.user_simulator_args['goals_file'] = \
a0_sim_config['goals_file']
if 'policy_file' in a0_sim_config:
self.user_simulator_args['policy_file'] = \
a0_sim_config['policy_file']
self.user_simulator = AgendaBasedUS(
self.user_simulator_args)
elif a0_sim_config['simulator'] == 'dtl':
if 'policy_file' in a0_sim_config:
self.user_simulator_args['policy_file'] = \
a0_sim_config['policy_file']
self.user_simulator = DTLUserSimulator(
self.user_simulator_args)
else:
raise ValueError(
'Error! Cannot start DAct-to-Language '
'simulator without a policy file!')
else:
# Fallback to agenda based simulator with default settings
self.user_simulator = AgendaBasedUS(
self.user_simulator_args)
# NLU Settings
if 'NLU' in self.configuration['AGENT_0'] and \
self.configuration['AGENT_0']['NLU'] and \
self.configuration['AGENT_0']['NLU']['nlu']:
nlu_args = dict(
zip(['ontology', 'database'],
[self.ontology, self.database]))
if self.configuration['AGENT_0']['NLU']['nlu'] == 'dummy':
self.nlu = DummyNLU(nlu_args)
elif self.configuration['AGENT_0']['NLU']['nlu'] == 'CamRest':
if self.configuration['AGENT_0']['NLU']['model_path']:
nlu_args['model_path'] = \
self.configuration['AGENT_0']['NLU']['model_path']
self.nlu = CamRestNLU(nlu_args)
else:
raise ValueError(
'Cannot find model_path in the config.')
# NLG Settings
if 'NLG' in self.configuration['AGENT_0'] and \
self.configuration['AGENT_0']['NLG'] and \
self.configuration['AGENT_0']['NLG']['nlg']:
if self.configuration['AGENT_0']['NLG']['nlg'] == 'dummy':
self.nlg = DummyNLG()
elif self.configuration['AGENT_0']['NLG']['nlg'] == 'CamRest':
if self.configuration['AGENT_0']['NLG']['model_path']:
self.nlg = CamRestNLG({
'model_path':
self.configuration['AGENT_0']['NLG']['model_path']
})
else:
raise ValueError(
'Cannot find model_path in the config.')
if self.nlg:
self.USE_NLG = True
# Retrieve agent role
if 'role' in self.configuration['AGENT_0']:
self.agent_role = self.configuration['AGENT_0']['role']
else:
raise ValueError(
'ConversationalAgent: No role assigned for agent {0} in '
'config!'.format(self.agent_id))
if self.agent_role == 'user':
if self.ontology and self.database:
self.goal_generator = GoalGenerator(ontology=self.ontology,
database=self.database)
else:
raise ValueError(
'Conversational Multi Agent (user): Cannot generate '
'goal without ontology and database.')
dm_args = dict(
zip([
'settings', 'ontology', 'database', 'domain', 'agent_id',
'agent_role'
], [
self.configuration, self.ontology, self.database, self.domain,
self.agent_id, self.agent_role
]))
dm_args.update(self.configuration['AGENT_0']['DM'])
self.dialogue_manager = DialogueManager.DialogueManager(dm_args)
def __init__(self,
ontology,
database,
agent_id=0,
agent_role='system',
alpha=0.25,
gamma=0.95,
epsilon=0.25,
alpha_decay=0.9995,
epsilon_decay=0.995,
epsilon_min=0.05,
warm_up_mode=False,
**kwargs):
"""
Initialize parameters and internal structures
:param ontology: the domain's ontology
:param database: the domain's database
:param agent_id: the agent's id
:param agent_role: the agent's role
:param alpha: the learning rate
:param gamma: the discount rate
:param epsilon: the exploration rate
:param alpha_decay: the learning rate discount rate
:param epsilon_decay: the exploration rate discount rate
"""
self.logger = logging.getLogger(__name__)
self.warm_up_mode = warm_up_mode
self.alpha = alpha
self.gamma = gamma
self.epsilon = epsilon
self.alpha_decay = alpha_decay
self.epsilon_decay = epsilon_decay
self.epsilon_min = epsilon_min
self.IS_GREEDY_POLICY = False
# TODO: Put these as arguments in the config
self.d_win = 0.0025
self.d_lose = 0.01
self.is_training = False
self.agent_id = agent_id
self.agent_role = agent_role
self.ontology = None
if isinstance(ontology, Ontology):
self.ontology = ontology
else:
raise ValueError('Unacceptable ontology type %s ' % ontology)
self.database = None
if isinstance(database, DataBase):
self.database = database
else:
raise ValueError('WoLF PHC DialoguePolicy: Unacceptable database '
'type %s ' % database)
self.Q = {}
self.pi = {}
self.mean_pi = {}
self.state_counter = {}
self.pp = pprint.PrettyPrinter(width=160) # For debug!
# System and user expert policies (optional)
self.warmup_policy = None
self.warmup_simulator = None
if self.agent_role == 'system':
# Put your system expert policy here
self.warmup_policy = \
HandcraftedPolicy.HandcraftedPolicy(self.ontology)
elif self.agent_role == 'user':
usim_args = dict(
zip(['ontology', 'database'], [self.ontology, self.database]))
# Put your user expert policy here
self.warmup_simulator = AgendaBasedUS(usim_args)
# Plato does not use action masks (rules to define which
# actions are valid from each state) and so training can
# be harder. This becomes easier if we have a smaller
# action set.
# Extract lists of slots that are frequently used
self.informable_slots = deepcopy(
list(self.ontology.ontology['informable'].keys()))
self.requestable_slots = deepcopy(
self.ontology.ontology['requestable'])
self.system_requestable_slots = deepcopy(
self.ontology.ontology['system_requestable'])
self.statistics = {'supervised_turns': 0, 'total_turns': 0}
self.hash2actions = {}
self.domain = setup_domain(self.ontology)
self.NActions = self.domain.NActions