def test_default_value(self):
builder = CategoricalTableBuilder("Burglary")
builder.add_row(ValueFactory.create(False), 0.8)
assert builder.build().get_prob(ValueFactory.none()) == pytest.approx(
0.199, abs=0.01)
builder.remove_row(ValueFactory.create(False))
assert builder.build().get_prob(ValueFactory.none()) == pytest.approx(
0.999, abs=0.01)
# assert node.hasProb(Assignment(), ValueFactory.none())
builder = CategoricalTableBuilder("Burglary")
builder.add_row(ValueFactory.create(False), 0.999)
assert builder.build().get_prob(ValueFactory.none()) == pytest.approx(
0.0, abs=0.01)
def get_prob_distrib(self, condition):
"""
Fills the cache with the resulting table for the given condition
:param condition: the condition for which to fill the cache
"""
builder = CategoricalTableBuilder(self._base_var + self._primes)
full_effects = list()
for inputVal in condition.get_values():
if isinstance(inputVal, Effect):
full_effects.extend(inputVal.get_sub_effects())
full_effect = Effect(full_effects)
values = full_effect.get_values(self._base_var)
if full_effect.is_non_exclusive(self._base_var):
add_val = ValueFactory.create(list(values.keys()))
builder.add_row(add_val, 1.0)
elif len(values) > 0:
total = 0.0
for f in values.values():
total += float(f)
for v in values.keys():
builder.add_row(v, values[v] / total)
else:
builder.add_row(ValueFactory.none(), 1.0)
return builder.build()
def __init__(self, arg1=None, arg2=None, arg3=1, arg4=True, arg5=False):
if isinstance(arg1, Template) and isinstance(
arg2, Template) and isinstance(arg3, int) and isinstance(
arg4, bool) and isinstance(arg5, bool):
variable, value, priority, exclusive, negated = arg1, arg2, arg3, arg4, arg5
"""
Constructs a new effect, with a variable label, value, and other arguments.
The argument "add" specifies whether the effect is mutually exclusive with
other effects. The argument "negated" specifies whether the effect includes
a negation.
:param variable: variable label
:param value: variable value
:param priority:the priority level (default is 1)
:param exclusive: whether distinct values are mutually exclusive or not
:param negated: whether to negate the effect or not.
"""
super(TemplateEffect, self).__init__(
str(variable),
ValueFactory.none() if value.is_under_specified() else
ValueFactory.create(str(value)), priority, exclusive, negated)
self._label_template = variable
self._value_template = value
else:
raise NotImplementedError()
def fill_slots(self, assignment):
"""
Fills the slots of the template, and returns the result of the function
evaluation. If the function is not a simple arithmetic expression,
"""
filled = super(ArithmeticTemplate, self).fill_slots(assignment)
if '{' in filled:
return filled
if ArithmeticTemplate.is_arithmetic_expression(filled):
try:
return StringUtils.get_short_form(
MathExpression(filled).evaluate())
# TODO: need to check exception handling
except Exception as e:
self.log.warning("cannot evaluate " + filled)
return filled
# handling expressions that manipulate sets
# (using + and - to respectively add/remove elements)
merge = ValueFactory.none()
for str_val in filled.split("+"):
negations = str_val.split("-")
merge = merge.concatenate(ValueFactory.create(negations[0]))
for negation in negations[1:]:
values = merge.get_sub_values()
old_value = ValueFactory.create(negation)
if old_value in values:
values.remove(ValueFactory.create(negation))
merge = ValueFactory.create(values)
return str(merge)
def remove_from_state(self, variable_id):
"""
Removes the variable from the dialogue state
:param variable_id: the node to remove
"""
with self._locks['remove_from_state']:
self.add_to_state(Assignment(variable_id, ValueFactory.none()))
def get_value(self, variable):
"""
Returns the value associated with the variable in the assignment, if one is
specified. Else, returns the none value.
:param variable: the variable
:return: the associated value
"""
return self._map.get(variable, ValueFactory.none())
def start(self):
"""
Adds an empty action to the dialogue system to start the interaction.
"""
empty_action = Assignment(self.system.get_settings().system_output, ValueFactory.none())
if self.system.is_paused():
self.system.get_state().add_to_state(empty_action)
else:
self.system.add_content(empty_action)
self.system.attach_module(RewardLearner)
def is_filled_by(self, assignment):
"""
Returns true if all slots are filled by the assignment. Else, returns false.
"""
# TODO: check whether this function has a bug or not.
for slot_key in self._slots:
value = assignment.get_value(slot_key)
if value == ValueFactory.none():
return False
return True
def is_empty(self):
"""
Returns true if the table is empty (or contains only a default assignment),
false otherwise
:return: true if empty, false otherwise
"""
if len(self._table) == 0:
return True
return len(self._table) == 1 and list(
self._table.keys())[0] == ValueFactory.none()
def is_default(self):
"""
Returns true if the assignment only contains none values for all variables,
and false if at least one has a different value.
:return: true if all variables have none values, false otherwise
"""
for variable in self._map.keys():
if self._map[variable] != ValueFactory.none():
return False
return True
def get_prob_distrib(self, condition):
"""
Returns the (unconditional) probability distribution P(X) given the conditional assignment.
:param condition: the conditional assignment
:return: the corresponding probability distribution
"""
if condition in self._table:
return self._table[condition]
else:
# TODO: check refactor > raise exception?
return SingleValueDistribution(self._head_var, ValueFactory.none())
def get_values(self, variables):
"""
Returns the list of values corresponding to a subset of variables in the
assignment (in the same order)
:param variables: the subset of variable labels
:return: the corresponding values
"""
values = []
for variable in variables:
values.append(self._map.get(variable, ValueFactory.none()))
return values
def create_default(variables):
"""
Creates an assignment with only none values for the variable labels given as argument.
:param args:
:param variables: the collection of variable labels
:return: the resulting default assignment
"""
assignment = Assignment()
for variable in variables:
assignment.add_pair(variable, ValueFactory.none())
return assignment
def get_prob(self, condition):
"""
Returns the probability of eq=true given the condition
:param condition: the conditional assignment
:return: the probability of eq=true
"""
predicted = None
actual = None
for input_var in condition.get_variables():
if input_var == self._base_var + "^p":
predicted = condition.get_value(input_var)
elif input_var == self._base_var + "'":
actual = condition.get_value(input_var)
elif input_var == self._base_var:
actual = condition.get_value(input_var)
if predicted is None or actual is None:
raise ValueError()
if predicted == ValueFactory.none() or actual == ValueFactory.none():
return EquivalenceDistribution.none_prob
elif predicted == actual:
return 1.0
elif isinstance(predicted, StringVal) and isinstance(actual, StringVal):
str1 = str(predicted)
str2 = str(actual)
if Template.create(str1).match(str2).is_matching() or Template.create(str2).match(str1).is_matching():
return 1.0
return 0.0
elif len(predicted.get_sub_values()) > 0 and len(actual.get_sub_values()) > 0:
vals0 = predicted.get_sub_values()
vals1 = actual.get_sub_values()
intersect = set(vals0)
intersect.intersection_update(vals1)
return float(len(intersect) / len(vals0))
else:
return 0.0
def get_values(self):
"""
Returns the possible outputs values given the input range in the parent nodes
(probability rule nodes)
:return: the possible values for the output
"""
values = set()
for rule in self._input_rules:
for effect in rule.get_effects():
if effect.is_non_exclusive(self._base_var):
return self.get_values_linearise()
set_values = set(effect.get_values(self._base_var).keys())
if set_values:
values.update(set_values)
else:
values.add(ValueFactory.none())
if len(values) == 0:
values.add(ValueFactory.none())
return values
def get_nodes_to_keep(state):
"""
Selects the set of variables to retain in the dialogue state.
:param state: the dialogue state
:return: the set of variable labels to keep
"""
nodes_to_keep = set()
for chance_node in state.get_chance_nodes():
if chance_node.get_id()[:2] == "=_" or chance_node.get_id(
)[-2:] == "^t" or chance_node.get_id()[-2:] == "^o":
continue
elif StatePruner.enable_reduction and isinstance(
chance_node.get_distrib(), AnchoredRule):
continue
elif len(chance_node.get_input_node_ids()
) < 3 and chance_node.get_nb_values() == 1 and list(
chance_node.get_values())[0] == ValueFactory.none():
continue
elif chance_node.get_id()[-2:] == "^p":
flag = False
for node_ids in chance_node.get_output_node_ids():
if node_ids[:2] == "=_":
flag = True
break
if flag:
continue
if not state.has_chance_node(chance_node.get_id() + "'"):
nodes_to_keep.add(chance_node.get_id())
if state.is_incremental(chance_node.get_id()):
for descendant_id in chance_node.get_descendant_ids():
if not state.has_chance_node(descendant_id):
continue
if state.has_chance_node(descendant_id + "'"):
continue
nodes_to_keep.add(descendant_id)
if chance_node.get_id() in state.get_parameter_ids(
) and not chance_node.has_descendant(
state.get_evidence().get_variables()):
for output_node in chance_node.get_output_nodes(ChanceNode):
if not isinstance(output_node.get_distrib(), AnchoredRule):
continue
nodes_to_keep.add(output_node.get_id())
return nodes_to_keep
def get_assignment(self):
"""
Returns the effect as an assignment of values. The variable labels are ended
by a prime character.
:return: the assignment of new values to the variables
"""
assignment = Assignment()
for effect in self._sub_effects:
if not effect._negated:
assignment.add_pair(effect.get_variable() + "'",
effect.get_value())
else:
assignment.add_pair(effect.get_variable() + "'",
ValueFactory.none())
return assignment
def sample(self, condition):
"""
Sample a head assignment from the distribution P(head|condition), given the
condition. If no assignment can be sampled (due to e.g. an ill-formed
distribution), returns an empty assignment.
:param condition: the condition
:return: the sampled assignment the condition
"""
if condition.size() != len(self._conditional_vars):
condition = condition.get_trimmed(self._conditional_vars)
subdistrib = self._table[condition]
if subdistrib is not None:
return subdistrib.sample()
# TODO: check refactor > raise exception?
return ValueFactory.none()
def perform_turn(self):
system_state = self.system.get_state()
output_var = self.system.get_settings().system_output
try:
system_action = ValueFactory.none()
if system_state.has_chance_node(output_var):
system_action = system_state.query_prob(output_var).get_best()
self.log.debug("Simulator input: %s" % system_action)
turn_performed = self.perform_turn(system_action)
repeat = 0
while not turn_performed and repeat < 5 and self.system.get_modules().contains(self):
turn_performed = self.perform_turn(system_action)
repeat += 1
except Exception as e:
self.log.debug("cannot update simulator: " + str(e))
def test_classical(self):
assert isinstance(ValueFactory.create(' blabla '), StringVal)
assert isinstance(ValueFactory.create('3'), DoubleVal)
assert isinstance(ValueFactory.create('3.6'), DoubleVal)
assert ValueFactory.create('3').get_double() == pytest.approx(
3.0, abs=0.0001)
assert isinstance(ValueFactory.create('[firstItem, secondItem, 3.6]'),
SetVal)
assert len(
ValueFactory.create(
'[firstItem, secondItem, 3.6]').get_sub_values()) == 3
assert ValueFactory.create(
'[firstItem, secondItem, 3.6]').get_sub_values() == {
ValueFactory.create('firstItem'),
ValueFactory.create('secondItem'),
ValueFactory.create(3.6)
}
assert isinstance(ValueFactory.create('[0.6, 0.4, 32]'), ArrayVal)
assert len(ValueFactory.create('[0.6, 0.4, 32]').get_array()) == 3
assert ValueFactory.create(
'[0.6, 0.4, 32]').get_array()[2] == pytest.approx(32, abs=0.0001)
assert isinstance(ValueFactory.create('True'), BooleanVal)
assert not ValueFactory.create('False').get_boolean()
assert ValueFactory.create('None') == ValueFactory.none()
assert not ValueFactory.create('firsttest').__lt__(
ValueFactory.create('firsttest'))
assert ValueFactory.create('firsttest').__lt__(
ValueFactory.create('secondTest'))
assert ValueFactory.create(3.0).__lt__(ValueFactory.create(5.0))
assert not ValueFactory.create(5.0).__lt__(ValueFactory.create(3.0))
assert (ValueFactory.create(5.0).__lt__(
ValueFactory.create('test'))) == (
ValueFactory.create('test').__lt__(ValueFactory.create(5.0)))
assert len(
ValueFactory.create('[test,[1,2],True]').get_sub_values()) == 3
assert ValueFactory.create('test') in ValueFactory.create(
'[test,[1,2],True]').get_sub_values()
assert ValueFactory.create('[1,2]') in ValueFactory.create(
'[test,[1,2],True]').get_sub_values()
assert ValueFactory.create('True') in ValueFactory.create(
'[test,[1,2],True]').get_sub_values()
assert len(
ValueFactory.create(
'[a1=test,a2=[1,2],a3=true]').get_sub_values()) == 3
def sample(self):
"""
Sample a value from the distribution. If no assignment can be sampled (due to
e.g. an ill-formed distribution), returns a none value.
:return: the sampled assignment
"""
if self._intervals is None:
if len(self._table) == 0:
self.log.warning("creating intervals for an empty table")
raise ValueError()
self._intervals = Intervals(self._table)
if self._intervals.is_empty():
self.log.warning("interval is empty, table: ", self._table)
return ValueFactory.none()
sample = self._intervals.sample()
return sample
def generate_xml(self):
"""
Generates the XML representation for the table, for the document doc.
:param doc: the XML document for which to generate the XML.
:return: XML reprensetation
"""
var = Element("variable")
var.set("id", self._variable.replace("'", ""))
for v in InferenceUtils.get_n_best(self._table,
len(self._table)).keys():
if v != ValueFactory.none():
value_node = Element("value")
if self._table[v] < 0.99:
value_node.set("prob",
StringUtils.get_short_form(self._table[v]))
value_node.text = str(v)
var.append(value_node)
return var
def get_values_linearise(self):
"""
Calculates the possible values for the output distribution via linearisation
(more costly operation, but necessary in case of add effects).
:return: the set of possible output values
"""
table = dict()
for i in range(len(self._input_rules)):
table[str(i)] = self._input_rules[i].get_effects()
combinations = InferenceUtils.get_all_combinations(table)
values = set()
for cond in combinations:
values.update(self.get_prob_distrib(cond).get_values())
if len(values) == 0:
values.add(ValueFactory.none())
return values
def create_table(self, variable):
"""
Extracts the values (along with their weight) specified in the effect.
:param variable: variable the variable
:return: the corresponding values
"""
values = dict()
max_priority = sys.maxsize
to_remove = set()
to_remove.add(ValueFactory.none())
for effect in self._sub_effects:
if effect.get_variable() == variable:
if effect._priority < max_priority:
max_priority = effect._priority
if effect._negated:
to_remove.add(effect.get_value())
for effect in self._sub_effects:
effect_variable = effect.get_variable()
if effect_variable == variable:
effect_value = effect.get_value()
if effect._priority > max_priority or effect._negated or effect_value in to_remove:
continue
if len(to_remove) > 1 and isinstance(effect_value, SetVal):
sub_values = effect_value.get_sub_values()
for r in to_remove:
if r in sub_values:
sub_values.remove(r)
effect_value = ValueFactory.create(sub_values)
if effect_value not in values:
values[effect_value] = effect._weight
else:
values[effect_value] += effect._weight
return values
def remove_spurious_nodes(reduced):
"""
Removes all non-necessary nodes from the dialogue state.
:param reduced: the reduced dialogue state
"""
for chance_node in set(reduced.get_chance_nodes()):
if len(chance_node.get_input_nodes()) == 0 and len(
chance_node.get_output_nodes()) == 0 and isinstance(
chance_node.get_distrib(),
CategoricalTable) and chance_node.get_prob(
ValueFactory.none()) > 0.99:
reduced.remove_node(chance_node.get_id())
continue
if isinstance(chance_node.get_distrib(),
EquivalenceDistribution) and len(
chance_node.get_input_node_ids()) == 0:
reduced.remove_node(chance_node.get_id())
chance_node.prune_values(StatePruner.value_pruning_threshold)
if chance_node.get_nb_values() == 1 and len(
chance_node.get_output_nodes()) > 0 and len(
reduced.get_incremental_vars()) == 0:
assignment = Assignment(chance_node.get_id(),
chance_node.sample())
for output_node in chance_node.get_output_nodes(ChanceNode):
if not isinstance(output_node.get_distrib(), AnchoredRule):
cur_distrib = output_node.get_distrib()
output_node.remove_input_node(chance_node.get_id())
if len(output_node.get_input_node_ids()) == 0:
output_node.set_distrib(
cur_distrib.get_prob_distrib(assignment))
else:
output_node.set_distrib(
cur_distrib.get_posterior(assignment))
def get_groundings(self, param):
"""
Returns the set of possible groundings for the given input assignment
:param param: the input assignment
:return: the set of possible (alternative) groundings for the condition
"""
ground_cond = BasicCondition(self, param)
groundings = RuleGrounding()
if len(ground_cond._variable.get_slots()) > 0:
for inputVar in param.get_variables():
m = ground_cond._variable.match(inputVar)
if m.is_matching():
new_input = Assignment([param, m])
spec_grounds = self.get_groundings(new_input)
spec_grounds.extend(m)
groundings.add(spec_grounds)
return groundings
filled_var = str(ground_cond._variable)
if len(ground_cond._template_value.get_slots()) > 0:
actual_value = param.get_value(str(ground_cond._variable))
groundings = ground_cond.get_groundings(actual_value)
groundings.remove_variables(param.get_variables())
groundings.remove_value(ValueFactory.none())
elif self._relation == Relation.IN and not param.contains_var(
filled_var):
values_coll = ground_cond._ground_value.get_sub_values()
groundings.extend(filled_var, values_coll)
elif not self.is_satisfied_by(param):
groundings.set_as_failed()
return groundings
def __init__(self, arg1=None, arg2=None):
if isinstance(arg1, str) and arg2 is None:
node_id = arg1
"""
Creates a new action node with a unique identifier, and no values
:param node_id: the node identifier
"""
super(ActionNode, self).__init__(node_id)
self._action_values = set()
self._action_values.add(ValueFactory.none())
elif isinstance(arg1, str) and isinstance(arg2, set):
node_id = arg1
action_values = arg2
"""
Creates a new action node with a unique identifier and a set of values
:param node_id: the node identifier
:param action_values: the values for the action
"""
super(ActionNode, self).__init__(node_id)
self._action_values = set()
self._action_values.update(action_values)
else:
raise NotImplementedError("UNDEFINED PARAMETERS")
def fill_slots(self, fillers):
"""
Fills the template with the given content, and returns the filled string. The
content provided in the form of a slot:filler mapping. For instance, given a
template: "my name is {name}" and a filler "name:Pierre", the method will
return "my name is Pierre".
:param fillers: the content associated with each slot.
:return: the string filled with the given content
"""
if len(self._slots) == 0:
return self._str_val
result = self._str_val
for slot_key in self._slots.keys():
value = fillers.get_value(slot_key)
if isinstance(value, CustomVal):
assert("{%s}" % slot_key == self._str_val) # only {custom_value} is allowed.
result = value.get_value()
elif value != ValueFactory.none():
str_val = str(value)
result = result.replace("{%s}" % slot_key, str_val)
return result