def _process_dialogs(self, data, exclude_domains=[]):
new_dialog = []
bod_utt = [BOS, BOD, EOS]
eod_utt = [BOS, EOD, EOS]
all_lens = []
all_dialog_lens = []
speaker_map = {'bot': SYS, 'user': USR}
for raw_dialog in data:
domain = raw_dialog['domain']
if domain in exclude_domains:
continue
dialog = [Pack(utt=bod_utt, speaker=0, meta=None)]
for utt_idx, utt in enumerate(raw_dialog['turns']):
author_type = 'bot' if utt_idx % 2 == 0 else 'user'
if self.config.include_domain:
utt = [BOS, speaker_map[author_type], domain] + self.tokenize(utt) + [EOS]
else:
utt = [BOS, speaker_map[author_type]] + self.tokenize(utt) + [EOS]
all_lens.append(len(utt))
dialog.append(Pack(utt=utt, speaker=speaker_map[author_type]))
if not hasattr(self.config, 'include_eod') or self.config.include_eod:
dialog.append(Pack(utt=eod_utt, speaker=0))
all_dialog_lens.append(len(dialog))
new_dialog.append(dialog)
print("Max utt len %d, mean utt len %.2f" % (np.max(all_lens), float(np.mean(all_lens))))
print("Max dialog len %d, mean dialog len %.2f" % (np.max(all_dialog_lens),
float(np.mean(all_dialog_lens))))
return new_dialog
def _process_dialog(self, data):
new_dialog = []
bod_utt = [BOS, BOD, EOS]
eod_utt = [BOS, EOD, EOS]
all_lens = []
all_dialog_lens = []
speaker_map = {'assistant': SYS, 'driver': USR}
for raw_dialog in data:
dialog = [Pack(utt=bod_utt, speaker=0, meta=None)]
for turn in raw_dialog['dialogue']:
utt = turn['data']['utterance']
utt = [BOS, speaker_map[turn['turn']]
] + self.tokenize(utt) + [EOS]
all_lens.append(len(utt))
dialog.append(Pack(utt=utt, speaker=turn['turn']))
if hasattr(self.config, 'include_eod') and self.config.include_eod:
dialog.append(Pack(utt=eod_utt, speaker=0))
all_dialog_lens.append(len(dialog))
new_dialog.append(dialog)
print("Max utt len %d, mean utt len %.2f" %
(np.max(all_lens), float(np.mean(all_lens))))
print("Max dialog len %d, mean dialog len %.2f" %
(np.max(all_dialog_lens), float(np.mean(all_dialog_lens))))
return new_dialog
def _process_dialogs(self, data):
new_dialog = []
bod_utt = [BOS, BOD, EOS]
all_lens = []
all_dialog_lens = []
speaker_map = {'bot': SYS, 'user': USR}
for raw_dialog in data:
domain = raw_dialog['domain']
dialog = [Pack(utt=bod_utt, speaker=USR, meta=None, domain=domain)]
for utt_idx, utt in enumerate(raw_dialog['turns']):
speaker = speaker_map['bot' if utt_idx % 2 == 0 else 'user']
if self.config.include_domain:
utt = [BOS, speaker, domain] + self.tokenize(utt) + [EOS]
else:
utt = [BOS, speaker] + self.tokenize(utt) + [EOS]
all_lens.append(len(utt))
if speaker == SYS:
dialog.append(
Pack(utt=utt, speaker=speaker, slots={}, domain=domain, kb={}))
else:
dialog.append(Pack(utt=utt, speaker=speaker, slots={}, domain=domain))
all_dialog_lens.append(len(dialog))
new_dialog.append(dialog)
print("Max utt len %d, mean utt len %.2f" % (
np.max(all_lens), float(np.mean(all_lens))))
print("Max dialog len %d, mean dialog len %.2f" % (
np.max(all_dialog_lens), float(np.mean(all_dialog_lens))))
return new_dialog
def _process_dialogue(self, data):
new_dlgs = []
all_sent_lens = []
all_dlg_lens = []
for key, raw_dlg in data.items():
norm_dlg = [Pack(speaker=USR, utt=[BOS, BOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size)]
for t_id in range(len(raw_dlg['db'])):
usr_utt = [BOS] + self.tokenize(raw_dlg['usr'][t_id]) + [EOS]
sys_utt = [BOS] + self.tokenize(raw_dlg['sys'][t_id]) + [EOS]
norm_dlg.append(Pack(speaker=USR, utt=usr_utt, db=raw_dlg['db'][t_id], bs=raw_dlg['bs'][t_id]))
norm_dlg.append(Pack(speaker=SYS, utt=sys_utt, db=raw_dlg['db'][t_id], bs=raw_dlg['bs'][t_id]))
all_sent_lens.extend([len(usr_utt), len(sys_utt)])
# To stop dialog
norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size))
# if self.config.to_learn == 'usr':
# norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size))
all_dlg_lens.append(len(raw_dlg['db']))
processed_goal = self._process_goal(raw_dlg['goal'])
new_dlgs.append(Pack(dlg=norm_dlg, goal=processed_goal, key=key))
self.logger.info('Max utt len = %d, mean utt len = %.2f' % (
np.max(all_sent_lens), float(np.mean(all_sent_lens))))
self.logger.info('Max dlg len = %d, mean dlg len = %.2f' % (
np.max(all_dlg_lens), float(np.mean(all_dlg_lens))))
return new_dlgs
def _process_dialog(self, data):
new_dialog = []
bod_utt = [BOS, BOD, EOS]
eod_utt = [BOS, EOD, EOS]
all_lens = []
all_dialog_lens = []
for raw_dialog, raw_act, raw_emotion in data:
dialog = [Pack(utt=bod_utt,
speaker=0,
meta=None)]
raw_dialog = raw_dialog.decode('ascii', 'ignore').encode()
raw_dialog = raw_dialog.split('__eou__')[0:-1]
raw_act = raw_act.split()
raw_emotion = raw_emotion.split()
for t_id, turn in enumerate(raw_dialog):
utt = turn
utt = [BOS] + self.tokenize(utt.lower()) + [EOS]
all_lens.append(len(utt))
dialog.append(Pack(utt=utt, speaker=t_id%2,
meta={'emotion': raw_emotion[t_id], 'act': raw_act[t_id]}))
if not hasattr(self.config, 'include_eod') or self.config.include_eod:
dialog.append(Pack(utt=eod_utt, speaker=0))
all_dialog_lens.append(len(dialog))
new_dialog.append(dialog)
print("Max utt len %d, mean utt len %.2f" % (
np.max(all_lens), float(np.mean(all_lens))))
print("Max dialog len %d, mean dialog len %.2f" % (
np.max(all_dialog_lens), float(np.mean(all_dialog_lens))))
return new_dialog
def _process_dialogue(self, data):
new_dlgs = []
all_sent_lens = []
all_dlg_lens = []
for session_id, raw_dlg in enumerate(data):
# norm_dlg = [Pack(speaker=USR, utt=[BOS, BOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size)]
norm_dlg = []
state_dict = self.init_state_dict()
state_dict_feed = [state_dict[k] for k in sorted(state_dict.keys())]
if len(raw_dlg)<2:
continue
for turn_dialog in raw_dlg:
utt = [BOS] + self.tokenize(turn_dialog['act_str']) + [EOS]
# utt = self.tokenize(turn_dialog['act_str'])
# utt = [BOT] + self.tokenize(turn_dialog['act_str']) + [EOT]
action_id = turn_dialog.get('action_id', 999)
action_turn = turn_dialog.get('turn',0)/2
if turn_dialog['Message.From']=='user':
norm_dlg.append(Pack(speaker=USR, utt=utt, raw_act_id=action_id, turn_num=action_turn, state_dict=state_dict_feed))
if self.config.state_type=='table':
state_dict = self.fill_in_state_dict(state_dict, turn_dialog['act_str'], 'user')
if self.config.input_type=='sat':
state_dict_feed = [self.assign_bucket(state_dict[k]) for k in sorted(state_dict.keys())]
else:
state_dict_feed = [state_dict[k] for k in sorted(state_dict.keys())]
else:
norm_dlg.append(Pack(speaker=SYS, utt=utt, raw_act_id=action_id, turn_num=action_turn, state_dict=state_dict_feed))
if self.config.state_type=='table':
state_dict = self.fill_in_state_dict(state_dict, turn_dialog['act_str'], 'agent')
if self.config.input_type=='sat':
state_dict_feed = [self.assign_bucket(state_dict[k]) for k in sorted(state_dict.keys())]
else:
state_dict_feed = [state_dict[k] for k in sorted(state_dict.keys())]
all_sent_lens.extend([len(utt)])
# To stop dialog
# print(norm_dlg)
if norm_dlg[-1]['speaker']==SYS:
# norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], raw_act_id=0))
norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], raw_act_id=0, turn_num=0, state_dict=state_dict_feed))
# norm_dlg.append(Pack(speaker=USR, utt=[BOT, EOD, EOT]))
# if self.config.to_learn == 'usr':
# norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size))
# processed_goal = self._process_goal(raw_dlg['goal'])
all_dlg_lens.append(len(raw_dlg))
new_dlgs.append(Pack(dlg=norm_dlg, key=raw_dlg[0]['session']))
self.logger.info('Max utt len = %d, mean utt len = %.2f' % (
np.max(all_sent_lens), float(np.mean(all_sent_lens))))
self.logger.info('Max dlg len = %d, mean dlg len = %.2f' % (
np.max(all_dlg_lens), float(np.mean(all_dlg_lens))))
return new_dlgs
def _process_dialog(self, data, laed_z_data):
new_dialog = []
all_lens = []
all_dialog_lens = []
for raw_dialog, laed_z in zip(data, laed_z_data):
domain = raw_dialog['scenario']['task']['intent']
kb_items = []
if raw_dialog['scenario']['kb']['items'] is not None:
for item in raw_dialog['scenario']['kb']['items']:
kb_items.append([KB] + self.tokenize(" ".join(
["{} {}".format(k, v) for k, v in item.items()])))
dialog = [
Pack(utt=[BOS, domain, BOD, EOS],
speaker=USR,
slots=None,
domain=domain)
]
for turn, laed_z_turn in zip(raw_dialog['dialogue'], laed_z):
utt = turn['data']['utterance']
slots = turn['data'].get('slots')
speaker = self.speaker_map[turn['turn']]
if self.config.include_domain:
utt = [BOS, speaker, domain] + self.tokenize(utt) + [EOS]
else:
utt = [BOS, speaker] + self.tokenize(utt) + [EOS]
all_lens.append(len(utt))
if speaker == SYS:
dialog.append(
Pack(utt=utt,
speaker=speaker,
slots=slots,
domain=domain,
kb=kb_items))
else:
dialog.append(
Pack(utt=utt,
speaker=speaker,
slots=slots,
domain=domain,
kb=[]))
self._process_domain_description(turn, domain, laed_z_turn)
all_dialog_lens.append(len(dialog))
new_dialog.append(dialog)
print("Max utt len %d, mean utt len %.2f" %
(np.max(all_lens), float(np.mean(all_lens))))
print("Max dialog len %d, mean dialog len %.2f" %
(np.max(all_dialog_lens), float(np.mean(all_dialog_lens))))
return new_dialog
def _to_id_corpus(self, name, data):
results = []
for dlg in data:
if len(dlg.dlg) < 1:
continue
id_dlg = []
for turn in dlg.dlg:
id_turn = Pack(utt=self._sent2id(turn.utt),
speaker=turn.speaker
)
id_dlg.append(id_turn)
results.append(Pack(dlg=id_dlg, key=dlg.key))
return results
def vae_train(self, batch_feed):
state_rep, action_rep = self.read_real_data_onehot_300(None, batch_feed)
state_action_rep = torch.cat([state_rep, action_rep],-1)
recon_batch, mu, logvar = self.vae(state_action_rep)
loss = self.vae_loss(recon_batch, state_action_rep, mu, logvar)
vae_loss = Pack(vae_loss=loss)
return vae_loss
def disc_train(self, sample_shape, batch_feed, fake_state_action=None):
batch_size = sample_shape[0]
# state_rep, action_rep = self.read_real_data(None, batch_feed)
state_rep, action_rep = self.read_real_data_onehot(None, batch_feed)
state_action = torch.cat([self.cast_gpu(state_rep), self.cast_gpu(action_rep)], -1)
embed_batch = self.vae.encode(state_action)
real_disc_v = self.discriminator(embed_batch.detach())
if fake_state_action is None:
fake_state_action = self.sample_step(sample_shape)
else:
fake_state_action = fake_state_action
if self.config.round_for_disc:
fake_disc_v = self.discriminator(fake_state_action.detach())
else:
fake_disc_v = self.discriminator(fake_state_action.detach())
rf_disc_v = torch.cat([real_disc_v, fake_disc_v], dim=0)
labels_one = torch.cat([torch.full((batch_size,),1.0), torch.full((batch_size,),0.0)])
labels_one = self.cast_gpu(labels_one)
# labels_one = self.cast_gpu(torch.FloatTensor([1] * batch_size + [0] * batch_size))
disc_loss = self.loss_BCE(rf_disc_v.view(-1), labels_one)
# disc_loss = BCELoss_double(rf_disc_v.view(-1), labels)
# disc_loss = - torch.mean(real_disc_v) + torch.mean(fake_disc_v)
disc_loss = Pack(disc_loss=disc_loss)
disc_acc = cal_accuracy(rf_disc_v.view(-1), labels_one)
return disc_loss, disc_acc
def gen_train(self, sample_shape):
state_action_rep =self.sample_step(sample_shape)
mean_dist = - (state_action_rep.mean(dim=0) - state_action_rep).pow(2).sum() * 0.00003
disc_v = self.discriminator(state_action_rep)
gen_loss = -torch.mean(torch.log(disc_v))
gen_loss = Pack(gen_loss= gen_loss, mean_dist = mean_dist)
return gen_loss, state_action_rep.detach()
def vae_train(self, batch_feed):
state_rep, action_rep = self.read_real_data_onehot(None, batch_feed)
state_action = torch.cat([self.cast_gpu(state_rep), self.cast_gpu(action_rep)], -1)
recon_batch = self.vae(state_action)
loss = self.autoencoder_loss(recon_batch, state_action)
vae_loss = Pack(vae_loss=loss)
return vae_loss
def disc_train(self, sample_shape, batch_feed, fake_state_action=None):
batch_size = sample_shape[0]
real_state_rep, action_data_feed = self.read_real_data(sample_shape, batch_feed)
real_disc_v = self.discriminator(real_state_rep, action_data_feed)
if fake_state_action is None:
fake_state_rep, fake_action_rep = self.sample_step(sample_shape)
else:
fake_state_rep, fake_action_rep = fake_state_action
fake_disc_v = self.discriminator(fake_state_rep.detach(), fake_action_rep.detach())
rf_disc_v = torch.cat([real_disc_v, fake_disc_v], dim=0)
true_label = 1.0
false_label = 0.0
labels_one = torch.cat([torch.full((batch_size,),true_label), torch.full((batch_size,),false_label)])
# labels_one = torch.cat([torch.full((batch_size,),1.0), torch.full((batch_size,),0.0)])
labels_one = self.cast_gpu(labels_one)
# labels_one = self.cast_gpu(torch.FloatTensor([1] * batch_size + [0] * batch_size))
disc_loss = self.loss_BCE(rf_disc_v.view(-1), labels_one) + self.discriminator.l2_norm()
# disc_loss = BCELoss_double(rf_disc_v.view(-1), labels)
# disc_loss = - torch.mean(real_disc_v) + torch.mean(fake_disc_v)
disc_loss = Pack(disc_loss=disc_loss)
disc_acc = cal_accuracy(rf_disc_v.view(-1), labels_one)
return disc_loss, disc_acc
def disc_train(self, sample_shape, batch_feed, fake_state_action=None):
batch_size = sample_shape[0]
real_state_rep, action_data_feed = self.read_real_data(sample_shape, batch_feed)
real_disc_v = self.discriminator(real_state_rep, action_data_feed)
if fake_state_action is None:
fake_state_rep, fake_action_rep = self.sample_step(sample_shape)
else:
fake_state_rep, fake_action_rep = fake_state_action
# if self.config.round_for_disc:
# fake_disc_v = self.discriminator(fake_state_rep.detach().round(), fake_action_rep.detach().round())
# else:
if np.random.random()<0.5:
fake_state_rep, fake_action_rep = real_state_rep.detach(), action_data_feed[:,torch.randperm(action_data_feed.size()[1])]
fake_disc_v = self.discriminator(fake_state_rep.detach(), fake_action_rep.detach())
rf_disc_v = torch.cat([real_disc_v, fake_disc_v], dim=0)
labels_one = torch.cat([torch.full((batch_size,),1.0), torch.full((batch_size,),0.0)])
labels_one = self.cast_gpu(labels_one)
# labels_one = self.cast_gpu(torch.FloatTensor([1] * batch_size + [0] * batch_size))
disc_loss = self.loss_BCE(rf_disc_v.view(-1), labels_one)
# disc_loss = BCELoss_double(rf_disc_v.view(-1), labels)
# disc_loss = - torch.mean(real_disc_v) + torch.mean(fake_disc_v)
disc_loss = Pack(disc_loss=disc_loss)
disc_acc = cal_accuracy(rf_disc_v.view(-1), labels_one)
return disc_loss, disc_acc
def _to_id_corpus(self, name, data):
results = []
for dlg in data:
if len(dlg.dlg) < 1:
continue
id_dlg = []
for turn in dlg.dlg:
id_turn = Pack(utt=self._sent2id(turn.utt),
speaker=turn.speaker,
raw_act_id=turn.raw_act_id,
turn_num=turn.turn_num,
state_dict=turn.state_dict
)
id_dlg.append(id_turn)
results.append(Pack(dlg=id_dlg, key=dlg.key))
return results
def disc_train(self, sample_shape, batch_feed, fake_state_action=None):
batch_size = sample_shape[0]
# state_rep, action_rep = self.read_real_data(None, batch_feed)
state_rep, action_rep = self.read_real_data_onehot_300(None, batch_feed)
# state_action = torch.cat([self.cast_gpu(state_rep), self.cast_gpu(action_rep)], -1)
embed_batch = self.vae.get_embed(self.cast_gpu(state_rep))
real_disc_v = self.discriminator(embed_batch.detach(), self.cast_gpu(action_rep))
if fake_state_action is None:
fake_state_action = self.sample_step(sample_shape)
else:
fake_state_action = fake_state_action
state_rep_f, action_rep_f = fake_state_action
if np.random.random()<0.5:
# print(action_rep.size())
# state_rep_f, action_rep_f = embed_batch.detach(), self.shuffle_action(action_rep)
state_rep_f, action_rep_f = embed_batch.detach(), action_rep[torch.randperm(action_rep.size()[0]), :]
# print(action_rep_f.size())
fake_disc_v = self.discriminator(state_rep_f.detach(), action_rep_f.detach())
rf_disc_v = torch.cat([real_disc_v, fake_disc_v], dim=0)
labels_one = torch.cat([torch.full((batch_size,),1.0), torch.full((batch_size,),0.0)])
labels_one = self.cast_gpu(labels_one)
# labels_one = self.cast_gpu(torch.FloatTensor([1] * batch_size + [0] * batch_size))
disc_loss = self.loss_BCE(rf_disc_v.view(-1), labels_one)
# disc_loss = BCELoss_double(rf_disc_v.view(-1), labels)
# disc_loss = - torch.mean(real_disc_v) + torch.mean(fake_disc_v)
disc_loss = Pack(disc_loss=disc_loss)
disc_acc = cal_accuracy(rf_disc_v.view(-1), labels_one)
return disc_loss, disc_acc
def gen_train(self, sample_shape):
# self.noise_factor = self.noise_factor * 0.999
state_rep, action_rep =self.sample_step(sample_shape)
disc_v = self.discriminator(state_rep, action_rep)
gen_loss = -torch.mean(torch.log(disc_v)) + self.generator.l2_norm()
gen_loss = Pack(gen_loss= gen_loss)
return gen_loss, (state_rep.detach(), action_rep.detach())
def flatten_dialog(self, data, backward_size):
results = []
for dialog in data:
for i in range(1, len(dialog) - 1):
e_id = i
s_id = max(0, e_id - backward_size)
response = dialog[i]
prev = dialog[i - 1]
next = dialog[i + 1]
response['utt'] = self.pad_to(self.max_utt_size,
response.utt,
do_pad=False)
prev['utt'] = self.pad_to(self.max_utt_size,
prev.utt,
do_pad=False)
next['utt'] = self.pad_to(self.max_utt_size,
next.utt,
do_pad=False)
contexts = []
for turn in dialog[s_id:e_id]:
turn['utt'] = self.pad_to(self.max_utt_size,
turn.utt,
do_pad=False)
contexts.append(turn)
results.append(
Pack(context=contexts,
response=response,
prev_resp=prev,
next_resp=next))
return results
def _process_domain_description(self, in_turn, in_domain, in_laed_z):
utt = in_turn['data']['utterance']
speaker = self.speaker_map[in_turn['turn']]
action = []
nlu_dict = in_turn['data'].get('nlu', {}).get('annotations', {})
if 'ner' in nlu_dict:
action += flatten_nlu_dict(nlu_dict['ner'])
if 'entity_linking' in nlu_dict:
for key, value in nlu_dict['entity_linking'].items():
actual_value = value
if type(value) == list:
actual_value = value[0]
action.append(actual_value['entity'])
if 'SUTime' in nlu_dict:
for item in nlu_dict['SUTime']:
action.append(item['text'])
action = ' '.join(action)
empty_action = not len(action.strip())
if self.config.include_domain:
utt = [BOS, speaker, in_domain] + self.tokenize(utt) + [EOS]
action = [BOS, speaker, in_domain] + self.tokenize(action) + [EOS]
else:
utt = [BOS, speaker] + self.tokenize(utt) + [EOS]
action = [BOS, speaker] + self.tokenize(action) + [EOS]
self.domain_descriptions.append(
Pack(domain=in_domain,
speaker=speaker,
utt=utt,
actions=action,
laed_z=in_laed_z,
empty_action=empty_action))
def _prepare_batch(self, selected_index):
rows = [self.data[idx] for idx in selected_index]
keys = []
metas = []
index = []
turn_id = []
state_onehot, state_convlab = [], []
action_id, action_id_binary, action_rep = [], [], []
for row in rows:
state_onehot.append(row['state_onehot'])
state_convlab.append(row['state_convlab'])
action_id.append(row['action_id'])
action_id_binary.append(row['action_id_binary'])
action_rep.append(row['action_rep_seg'])
# state = np.array(state)
# action = np.array(action)
return Pack(action_id=action_id,
state_onehot=state_onehot,
state_convlab=state_convlab,
action_id_binary=action_id_binary,
action_rep_seg=action_rep)
def greedy_forward(self, data_feed):
ctx_lens = data_feed['context_lens']
batch_size = len(ctx_lens)
ctx_utts = self.np2var(data_feed['contexts'], LONG)
out_utts = self.np2var(data_feed['outputs'], LONG)
# context encoder
c_inputs = self.utt_encoder(ctx_utts)
c_outs, c_last = self.ctx_encoder(c_inputs, ctx_lens)
c_last = c_last.squeeze(0)
# prior network
py_logits = self.p_y(c_last).view(-1, self.config.k)
# map sample to initial state of decoder
sample_y, y_id = self.greedy_cat_connector(py_logits, self.use_gpu, return_max_id=True)
# pack attention context
attn_inputs = None
labels = out_utts[:, 1:].contiguous()
sample_y = sample_y.view(-1, self.config.k * self.config.y_size)
dec_init_state = self.c_init_connector(sample_y) + c_last.unsqueeze(0)
# decode
dec_outs, dec_last, dec_ctx = self.decoder(batch_size, out_utts[:, 0:-1], dec_init_state,
attn_context=attn_inputs,
mode=TEACH_FORCE, gen_type="greedy",
beam_size=self.config.beam_size)
enc_dec_nll = self.nll_loss(dec_outs, labels)
return Pack(nll=enc_dec_nll)
def _read_file(domain):
with open(
os.path.join(path,
'domain_descriptions/{}.tsv'.format(domain)),
'rb') as f:
lines = f.readlines()
for l in lines[1:]:
tokens = l.split('\t')
if tokens[2] == "":
break
utt = tokens[1]
speaker = tokens[0]
action = tokens[3]
if self.config.include_domain:
utt = [BOS, self.speaker_map[speaker], domain
] + self.tokenize(utt) + [EOS]
action = [BOS, self.speaker_map[speaker], domain
] + self.tokenize(action) + [EOS]
else:
utt = [BOS, self.speaker_map[speaker]
] + self.tokenize(utt) + [EOS]
action = [BOS, self.speaker_map[speaker]
] + self.tokenize(action) + [EOS]
seed_responses.append(
Pack(domain=domain,
speaker=speaker,
utt=utt,
actions=action))
def _to_id_corpus(self, name, data, use_black_list, domains):
results = []
kick_cnt = 0
domain_cnt = []
for dialog in data:
if len(dialog) < 1:
continue
domain = dialog[0].domain
if domain not in domains:
continue
should_filter = np.random.rand() < self.black_ratio
if use_black_list and self.black_domains \
and domain in self.black_domains \
and should_filter:
kick_cnt += 1
continue
temp = []
# convert utterance and feature into numeric numbers
for turn in dialog:
id_turn = Pack(
utt=self._sent2id(turn.utt),
utt_raw=turn.utt,
speaker=turn.speaker,
domain=turn.domain,
domain_id=self.rev_vocab[domain],
meta=turn.get('meta'),
kb=[self._sent2id(item) for item in turn.get('kb', [])],
kb_raw=turn.get('kb', []))
temp.append(id_turn)
results.append(temp)
domain_cnt.append(domain)
self.logger.info("Filter {} samples from {}".format(kick_cnt, name))
self.logger.info(Counter(domain_cnt).most_common())
return results
def _read_domain_descriptions_kb(self, path):
# read all domains
seed_responses = []
with open(os.path.join(path, 'kb_seed_data.json')) as kb_data_in:
kb_data = json.load(kb_data_in)
for domain, turns in kb_data.items():
for turn in turns:
if self.config.include_domain:
utt = [BOS, self.speaker_map[turn['agent']], domain
] + self.tokenize(turn['utterance']) + [EOS]
kb = [BOS, self.speaker_map[turn['agent']], domain
] + self.tokenize(turn['kb']) + [EOS]
else:
utt = [BOS, self.speaker_map[turn['agent']]
] + self.tokenize(turn['utterance']) + [EOS]
kb = [BOS, self.speaker_map[turn['agent']]
] + self.tokenize(turn['kb']) + [EOS]
seed_responses.append(
Pack(domain=domain,
speaker=turn['agent'],
utt=utt,
actions=kb))
return seed_responses
def vae_train(self, batch_feed):
# state_rep, action_rep = self.read_real_data_onehot(None, batch_feed)
state_rep, action_rep = self.read_real_data_onehot_300(None, batch_feed)
state_rep = self.cast_gpu(state_rep)
recon_batch, mu, logvar = self.vae(state_rep)
loss = self.vae_loss(recon_batch, state_rep, mu, logvar)
vae_loss = Pack(vae_loss=loss)
return vae_loss
def qzx_forward_simple(self, out_utts):
# this func will be used to extract latent action given original actions later in whole pipeline
output_embedding = self.x_embedding(out_utts)
x_outs, x_last = self.x_encoder(output_embedding)
x_last = x_last.transpose(0, 1).contiguous().view(-1, self.config.dec_cell_size)
qy_logits = self.q_y(x_last).view(-1, self.config.k)
log_qy = F.log_softmax(qy_logits, dim=1)
return Pack(qy_logits=qy_logits, log_qy=log_qy)
def _to_id_corpus(self, data):
results = []
for line in data:
id_turn = Pack(utt=self._sent2id(line.utt),
speaker=line.speaker,
meta=line.get('meta'))
results.append(id_turn)
return results
def gen_train(self, sample_shape):
state_rep, action_rep =self.sample_step(sample_shape)
mean_dist_1 = - (state_rep.mean(dim=0) - state_rep).pow(2).mean() * self.config.sim_factor
mean_dist_2 = - (action_rep.mean(dim=0) - action_rep).pow(2).mean() * self.config.sim_factor
mean_dist = mean_dist_1 + mean_dist_2
# mean_dist = mean_dist_2
disc_v = self.discriminator(state_rep, action_rep)
gen_loss = -torch.mean(torch.log(disc_v))
gen_loss = Pack(gen_loss= gen_loss, mean_dist = mean_dist)
return gen_loss, (state_rep.detach(), action_rep.detach())
def flatten_dialog(self, data):
results = []
for dialog in data:
for i in range(len(dialog) - 1):
response = dialog[i]
response['utt'] = self.pad_to(self.max_utt_size,
response.utt,
do_pad=False)
results.append(Pack(response=response))
return results
def _process_data(self, data):
all_text = []
all_lens = []
for line in data:
tokens = [BOS] + self.tokenize(line)[1:] + [EOS]
all_lens.append(len(tokens))
all_text.append(Pack(utt=tokens, speaker=0))
print("Max utt len %d, mean utt len %.2f" %
(np.max(all_lens), float(np.mean(all_lens))))
return all_text
