def sample(self, sess, words, vocab, num=200, start=u'从前', sampling_type=1):
state = sess.run(self.cell.zero_state(1, tf.float32))
attention_states = sess.run(tf.truncated_normal([1, self.attn_length, self.attn_size],stddev=0.1,dtype=tf.float32))
if type(start) is str:
start = unicode(start,encoding='utf-8')
for word in start:
x = np.zeros((1, 1))
x[0, 0] = words[word]
if self.args.attention is True:
feed = {self.input_data: x, self.initial_state:state, self.attention_states:attention_states}
[probs, state, attention_states] = sess.run([self.probs, self.final_state, self.attention_states], feed)
else:
feed = {self.input_data: x, self.initial_state:state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
ret = start
word = start[-1]
for n in range(num):
x = np.zeros((1, 1))
x[0, 0] = words[word]
if self.args.attention is True:
feed = {self.input_data: x, self.initial_state:state, self.attention_states:attention_states}
[probs, state, attention_states] = sess.run([self.probs, self.final_state, self.attention_states], feed)
else:
feed = {self.input_data: x, self.initial_state:state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
p = probs[0]
sample = random_pick(p,word,sampling_type)
pred = vocab[sample]
ret += pred
word = pred
return ret
def confirm_generator(self, available_script, wording_list, p_list):
"""
Get a confirm scene based on dialog history
"""
assert len(self.attr_list) == len(self.entity_list)
assert len(wording_list) == len(p_list)
# obtain basic scene
attr = self.attr_list[-1]
entity = self.entity_list[-1]
wording = random_pick(wording_list, p_list)
scene_name = " ".join(
["confirm", str(entity), "attr=" + attr, wording])
scene_content = list()
for turn in available_script[attr][wording]:
if type(turn) is list:
scene_content.append(random.choice(turn))
else:
scene_content.append(turn)
scene_content = [
turn.replace("entity", str(entity)) for turn in scene_content
]
# Now we make context important
if wording == "lack_entity" and self.resolve_context(self.entity_list):
scene_content = [scene_content[0], scene_content[-1]]
else:
pass
return scene_name, scene_content
def move(self):
node = utils.random_pick(self.get_active_nodes())
info = {'prestate': copy.copy(node)}
rule = node.move()
info['rule'] = rule
info['poststate'] = copy.copy(node)
return info
def move(self):
'''Pick up a random active rule in the node
and make a move according to its assignment.
'''
active_rule_names = self.get_active_rules()
assert active_rule_names, 'There is no active rule in the node.'
rule_name = utils.random_pick(active_rule_names)
is_active, variables = self.get_rule(rule_name)()
assert is_active
self.assign(**variables)
return rule_name
def sample(self,
sess,
words,
vocab,
num=200,
start=u'我们',
sampling_type=1):
state = sess.run(self.cell.zero_state(1, tf.float32))
# 在抽样生成的时候，我们首先需要一个种子序列，同时在第一步的时候，我们需要向网络传入一个0的初始状态，
# 并通过种子序列的第一个字得到下一个隐含状态，然后再结合种子的第二个字传入下一个隐含状态，直到种子序列传入完毕
for word in start:
x = np.zeros((1, 1))
x[0, 0] = words[word]
if not self.args.attention:
feed = {self.input_data: x, self.initial_state: state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
else:
# TO BE UPDATED
attention_states = sess.run(
build_weight(
[self.args.batch_size, self.attn_length, self.attn_size],
name='attention_states'))
feed = {
self.input_data: x,
self.initial_state: state,
self.attention_states: attention_states
}
[probs, state] = sess.run([self.probs, self.final_state], feed)
# 其中start是种子序列，attention是判断是否加入了注意力机制。
ret = start
word = start[-1]
for n in range(num):
x = np.zeros((1, 1))
x[0, 0] = words[word]
if not self.args.attention:
feed = {self.input_data: x, self.initial_state: state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
else:
feed = {
self.input_data: x,
self.initial_state: state,
self.attention_states: attention_states
}
[probs, state] = sess.run([self.probs, self.final_state], feed)
p = probs[0]
sample = random_pick(p, word, sampling_type)
pred = vocab[sample]
ret += pred
word = pred
return ret
def sample(self,
sess,
words,
vocab,
num=200,
start=u'我们',
sampling_type=1):
state = sess.run(self.cell.zero_state(1, tf.float32))
for word in start:
x = np.zeros((1, 1))
x[0, 0] = words[word]
if not self.args.attention:
feed = {self.input_data: x, self.initial_state: state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
else:
# TO BE UPDATED
attention_states = sess.run(
build_weight([
self.args.batch_size, self.attn_length, self.attn_size
],
name='attention_states'))
feed = {
self.input_data: x,
self.initial_state: state,
self.attention_states: attention_states
}
[probs, state] = sess.run([self.probs, self.final_state], feed)
ret = start
word = start[-1]
for n in range(num):
x = np.zeros((1, 1))
x[0, 0] = words[word]
if not self.args.attention:
feed = {self.input_data: x, self.initial_state: state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
else:
feed = {
self.input_data: x,
self.initial_state: state,
self.attention_states: attention_states
}
[probs, state] = sess.run([self.probs, self.final_state], feed)
p = probs[0]
sample = random_pick(p, word, sampling_type)
pred = vocab[sample]
ret += pred
word = pred
return ret
def test_06(self):
""" Create multiple bootable and non_bootable volumes """
image_id = utils.image_id(self.tenant_id, self.auth_token, config.image_name)
my_list = [utils.volume_create(tenant_id=self.tenant_id, auth_token=self.auth_token,
name="", size=config.non_bootable_volume_size, image_id=""),
utils.volume_create(self.tenant_id, self.auth_token,
config.bootable_volume_name, config.bootable_volume_size, image_id)]
func = utils.random_pick(my_list)
for i in range(4):
volume = (next(func))
volume_id = volume["volume"]["id"]
is_available = utils.is_volume_available(self.tenant_id, self.auth_token, volume_id)
volume_details = utils.volume_details(self.tenant_id, self.auth_token, volume_id)
self.assertEquals("available", volume_details["volume"]["status"],"volume is not in available state")
def generate_line(self, end_word):
"""
Generates a single line, given the last word.
"""
emission = []
num_syllables = 0
last = self.inverted_vocab[end_word]
state = random_pick(range(self.hidden), \
np.divide(self.O[:, last], np.sum(self.O[:, last])))
num_syllables += len(self.inverted_meter[end_word][0].split(','))
emission.append(end_word)
prev_word = end_word
while num_syllables < 10:
# Sample next observation.
next_probs = self.filter_next(num_syllables, prev_word,
self.O[state, :])
next_obs = random_pick(range(self.obs), next_probs)
next_word = self.vocab[next_obs]
# If only ' or - show up in word, then skip
if not re.search('[a-z]+', next_word):
continue
emission.append(next_word)
stresses = self.inverted_meter[next_word][0].split(',')
num_syllables += len(stresses)
prev_word = next_word
next_state = random_pick(range(self.hidden), self.A[state, :])
state = next_state
return emission[::-1]
def generate_nodes(self, emulation_id):
connection = client.connect(g.db)
cursor = connection.cursor()
cursor.execute("""SELECT id, probability FROM cities""")
cities = cursor.fetchall()
cursor.execute("""
SELECT nodes_qty
FROM emulations
WHERE id='{id}'
""".format(id=emulation_id))
nodes_qty = cursor.fetchone()[0]
print '--> Generating {nodes_qty} nodes'.format(nodes_qty=nodes_qty)
location_ids = []
probabilities = []
for city in cities:
location_ids.append(city[0])
probabilities.append(city[1])
nodes_ids = []
for _ in xrange(nodes_qty):
location_id = random_pick(location_ids, probabilities)
node_id = str(uuid.uuid4())
cursor.execute("""
INSERT INTO nodes(
id,
location_id,
emulation_id)
VALUES(
'{id}',
'{location_id}',
'{emulation_id}'
)
""".format(id=node_id,
location_id=location_id,
emulation_id=emulation_id))
nodes_ids.append(node_id)
cursor.execute("""REFRESH TABLE nodes""")
cursor.close()
connection.close()
print '--> Generating nodes complete'
return nodes_ids
def get_compared_coordinate(self, available_coordinate, p_list):
"""
In the compare scene, we choose which target to be compared
"""
row = self.coordinate[0]
available_coordinate_in_row = list()
p_list_in_row = list()
for coordinate, p in zip(available_coordinate, p_list):
if coordinate[0] == row:
available_coordinate_in_row.append(coordinate)
p_list_in_row.append(p)
p_list_in_row = [math.exp(5 * p) for p in p_list_in_row]
# choose the entity to compare
compared_coordinate = random_pick(available_coordinate_in_row,
p_list_in_row)
return compared_coordinate
def pick_words(cate: str, lst: list, num: int) -> list:
"""
Add category to random picked words.
Parameters
----------
cate: str
category
lst, num:
See `random_pick`.
Returns
-------
List with category.
Notes
-----
"""
wlist = random_pick(lst, num)
res = [(w, cate) for w in wlist if w not in IGNORE]
return res
def episode_generator(self, user_concern_attr, user_concern_entity):
"""
Control the flow of pre_sales
"""
self.init_episode(user_concern_attr, user_concern_entity)
terminal = False
while not terminal:
# First, we try to explore a new point
available_coordinate, p_list = self.coordinate_p()
self.coordinate = random_pick(available_coordinate, p_list)
self.matrix[self.coordinate] = 1
coordinate_index = available_coordinate.index(self.coordinate)
del available_coordinate[coordinate_index]
del p_list[coordinate_index]
# Then we see if some intents are not feasible for current explored point
# todo: if new attributes are added in KB, this part may need rewrite!
current_available_intent = copy.deepcopy(self.available_intent)
if self.user_concern_attr[self.coordinate[0]] not in COMPARE_PERMITTED_ATTR \
and "compare" in current_available_intent:
current_available_intent.remove("compare")
if np.sum(self.matrix[self.coordinate[0]]) == len(user_concern_entity) \
and "compare" in current_available_intent:
current_available_intent.remove("compare")
if sum(self.matrix[self.coordinate[0]]) == 1 \
and "confirm" in current_available_intent:
current_available_intent.remove("confirm")
if "confirm" in current_available_intent:
del_confirm = True
for qa_coordinate in self.intent_coordinate_dict["qa"]:
if self.coordinate[0] == qa_coordinate[0]:
del_confirm = False
break
if del_confirm:
current_available_intent.remove("confirm")
# Next we sample a intent and get according dialog script
available_intent_p_dict = filter_p_dict(current_available_intent,
self.intent_p_dict)
self.intent = random_pick(list(available_intent_p_dict.keys()),
list(available_intent_p_dict.values()))
self.history_intent.append(self.intent)
available_script = self.script[self.intent]
self.intent_coordinate_dict[self.intent].append(self.coordinate)
# generate specific script
if self.intent in ["qa", "confirm"]:
move = self.calculate_move()
available_grammar_p_dict = copy.deepcopy(
self.grammar_p_dict[self.intent][move])
self.attr_list.append(
self.user_concern_attr[self.coordinate[0]])
self.entity_list.append(
self.user_concern_entity[self.coordinate[1]])
if self.intent == "qa":
scene_name, scene_content = self.qa_generator(
available_script,
list(available_grammar_p_dict.keys()),
list(available_grammar_p_dict.values()))
else:
scene_name, scene_content = self.confirm_generator(
available_script,
list(available_grammar_p_dict.keys()),
list(available_grammar_p_dict.values()))
self.previous_coordinate2 = self.previous_coordinate1
self.previous_coordinate1 = self.coordinate
else:
# Get the compared coordinate
compared_coordinate = self.get_compared_coordinate(
available_coordinate, p_list)
move = self.calculate_move(
compared_coordinate=compared_coordinate)
self.matrix[compared_coordinate] = 1
available_grammar_p_dict = copy.deepcopy(
self.grammar_p_dict[self.intent][move])
available_grammar_p_dict = filter_p_dict(
list(available_script[self.user_concern_attr[
self.coordinate[0]]].keys()), available_grammar_p_dict)
self.attr_list.append(
self.user_concern_attr[compared_coordinate[0]])
self.attr_list.append(
self.user_concern_attr[self.coordinate[0]])
self.entity_list.append(
self.user_concern_entity[self.coordinate[1]])
self.entity_list.append(
self.user_concern_entity[compared_coordinate[1]])
scene_name, scene_content = self.compare_generator(
available_script, list(available_grammar_p_dict.keys()),
list(available_grammar_p_dict.values()))
self.previous_coordinate2 = compared_coordinate
self.previous_coordinate1 = self.coordinate
self.episode_script[scene_name] = scene_content
if np.sum(self.matrix
) == self.matrix.shape[0] * self.matrix.shape[1]:
self.episode_script["pre_sales_end"] = []
terminal = True
return self.episode_script
def compare_generator(self, available_script, wording_list, p_list):
"""
Get a confirm scene based on dialog history
"""
assert len(self.attr_list) == len(self.entity_list)
assert len(wording_list) == len(p_list)
assert self.attr_list[-1] == self.attr_list[
-2], "We should compare the same attribute!"
# obtain basic scene
attr = self.attr_list[-1]
entity1 = self.entity_list[-2]
entity2 = self.entity_list[-1]
# A trick to reduce agent response space
# Note: we modify scene_name and last turn of current scene only!
if entity2 in self.entity_list[:-2] and self.entity_list[:-2].index(
entity2) < self.entity_list.index(entity1):
replace_flag = True
else:
replace_flag = False
wording = random_pick(wording_list, p_list)
if replace_flag is False:
scene_name = " ".join([
"compare",
str(entity1),
str(entity2), "attr=" + attr, wording
])
else:
scene_name = " ".join([
"compare",
str(entity2),
str(entity1), "attr=" + attr, wording
])
scene_content = list()
for turn in available_script[attr][wording]:
if type(turn) is list:
scene_content.append(random.choice(turn))
else:
scene_content.append(turn)
last_turn = scene_content[-1]
scene_content = scene_content[:-1]
scene_content = [
turn.replace("entity1", str(entity1)) for turn in scene_content
]
scene_content = [
turn.replace("entity2", str(entity2)) for turn in scene_content
]
if replace_flag is False:
last_turn = last_turn.replace("entity1", str(entity1))
last_turn = last_turn.replace("entity2", str(entity2))
else:
last_turn = last_turn.replace("entity1", str(entity2))
last_turn = last_turn.replace("entity2", str(entity1))
scene_content.append(last_turn)
# Now we make context important
if wording == "lack_attribute":
if self.resolve_context(self.attr_list):
scene_content = [scene_content[0], scene_content[-1]]
else:
scene_content[0] = re.sub(r"[那么]", "", scene_content[0])
return scene_name, scene_content
def __init__(self, *args, **kwargs):
super(CentroidAlgNode, self).__init__(*args, **kwargs)
self.p = random_pick(list(self.neighbours) + [self])
def get_random_neighbour(self): return utils.random_pick(list(self.neighbours))
score = ((demand['continuous_freq'] + demand['interval_freq'])
/ (2*DEMAND_NF) * demand['publish_freq'])
res[cate][post] = score
return res
if __name__ == '__main__':
import pprint
from init import generate
from utils import random_pick
user_chosen_dict = generate()
like = user_chosen_dict['like']
cando = user_chosen_dict['cando']
choose_like = random_pick(like, 7)
choose_cando = random_pick(cando, 7)
print("Choose Like: ", choose_like)
print("Choose Cando: ", choose_cando)
res1 = get_duty_cate_score(choose_like)
res2 = get_require_post_score(choose_cando)
res3 = get_demand_post_score(choose_cando)
res = choose(user_chosen_dict)
pprint.pprint(("duty:", res1))
pprint.pprint(("requre:", res2))
pprint.pprint(("demand:", res3))
res3_2 = (get_demand_post_score_from_require_res(res2))
pprint.pprint(res3_2)
def episode_generator(self):
self.init_episode()
intent = random_pick(list(self.available_intent_p_dict.keys()),
list(self.available_intent_p_dict.values()))
intent_list = intent.split("_")
if len(intent_list) == 1:
intent = intent_list[0]
scene_name, scene_content = self.scene_generator(intent)
self.episode_script[scene_name] = scene_content
else:
intent1 = intent_list[0]
intent2 = intent_list[1]
scene_name1, scene_content1 = self.scene_generator(intent1)
if intent1 == "consult" and intent2 == "refund":
scene_name2, scene_content2 = self.scene_generator(
intent2, "withConsult")
else:
scene_name2, scene_content2 = self.scene_generator(
intent2, "color", "withExchange")
self.episode_script[scene_name1] = scene_content1
self.episode_script[scene_name2] = scene_content2
# In such case, we should delete exchange scene
if intent1 == "exchange" and intent2 == "exchange" and scene_name1.find("verbose") != -1 \
and scene_name1.find("3") == -1 or scene_name1.find("color") != -1:
key = list(self.episode_script.keys())[-1]
del self.episode_script[key]
# In such case, we should delete refund
if intent1 == "consult" and intent2 == "refund" and scene_name1.find("verbose3") != -1 \
and "os" not in scene_name1.split(" "):
key = list(self.episode_script.keys())[-1]
del self.episode_script[key]
# We can reorganize the exchange&exchange script to get more variable dialog flow
if len(self.episode_script
) == 2 and intent1 == "exchange" and intent2 == "exchange":
a_index = list()
for i, turn in enumerate(scene_content1):
if turn in [
"$name$", "$phone$", "$address$", "orderNumber"
]:
a_index.append(i)
rule = random.choice(REORGANIZE_RULES)
if rule == "mix":
i = random.choice(a_index)
self.episode_script[scene_name1][i] = " ".join([
self.episode_script[scene_name1][i],
self.episode_script[scene_name2][0]
])
remain_len = len(self.episode_script[scene_name2]) - 1
for x in range(i + 1, i + remain_len):
self.episode_script[scene_name1].insert(
x, self.episode_script[scene_name2][x - i])
self.episode_script[scene_name1][
i + remain_len] = " ".join([
self.episode_script[scene_name2][remain_len],
self.episode_script[scene_name1][i + remain_len]
])
key = list(self.episode_script.keys())[-1]
del self.episode_script[key]
self.episode_script[" ".join(
[scene_name1,
scene_name2])] = self.episode_script[scene_name1]
del self.episode_script[scene_name1]
elif rule == "insert":
i = random.choice(a_index)
remain_len = len(self.episode_script[scene_name2])
for x in range(i, i + remain_len - 1):
self.episode_script[scene_name1].insert(
x, self.episode_script[scene_name2][x - i])
self.episode_script[scene_name1].insert(
i + remain_len - 1, " ".join([
self.episode_script[scene_name2][remain_len - 1],
self.episode_script[scene_name1][i - 1]
]))
key = list(self.episode_script.keys())[-1]
del self.episode_script[key]
self.episode_script[" ".join(
[scene_name1,
scene_name2])] = self.episode_script[scene_name1]
del self.episode_script[scene_name1]
return self.episode_script