def __init__(self):
self.reader = MultiWozReader()
if len(cfg.cuda_device) == 1:
self.m = DAMD(self.reader)
else:
m = DAMD(self.reader)
self.m = torch.nn.DataParallel(m, device_ids=cfg.cuda_device)
# print(self.m.module)
self.evaluator = MultiWozEvaluator(self.reader) # evaluator class
if cfg.cuda: self.m = self.m.cuda() #cfg.cuda_device[0]
self.optim = Adam(lr=cfg.lr,
params=filter(lambda x: x.requires_grad,
self.m.parameters()),
weight_decay=5e-5)
self.base_epoch = -1
if cfg.limit_bspn_vocab:
self.reader.bspn_masks_tensor = {}
for key, values in self.reader.bspn_masks.items():
v_ = cuda_(torch.Tensor(values).long())
self.reader.bspn_masks_tensor[key] = v_
if cfg.limit_aspn_vocab:
self.reader.aspn_masks_tensor = {}
for key, values in self.reader.aspn_masks.items():
v_ = cuda_(torch.Tensor(values).long())
self.reader.aspn_masks_tensor[key] = v_
def __init__(self, device):
self.device = device
# initialize tokenizer
self.tokenizer = GPT2Tokenizer.from_pretrained(cfg.gpt_path)
# cfg.tokenizer = tokenizer
# initialize multiwoz reader
self.reader = MultiWozReader(self.tokenizer)
# create model: gpt2
self.model = GPT2LMHeadModel.from_pretrained(cfg.gpt_path)
if cfg.mode == 'train':
self.model.resize_token_embeddings(len(self.tokenizer))
self.model.to(self.device) # single gpu
#
self.evaluator = MultiWozEvaluator(self.reader)
if cfg.save_log and cfg.mode == 'train':
self.tb_writer = SummaryWriter(log_dir='./log')
else:
self.tb_writer = None
class Modal(object):
def __init__(self, device):
self.device = device
# initialize tokenizer
self.tokenizer = GPT2Tokenizer.from_pretrained(cfg.gpt_path)
# cfg.tokenizer = tokenizer
# initialize multiwoz reader
self.reader = MultiWozReader(self.tokenizer)
# create model: gpt2
self.model = GPT2LMHeadModel.from_pretrained(cfg.gpt_path)
if cfg.mode == 'train':
self.model.resize_token_embeddings(len(self.tokenizer))
self.model.to(self.device) # single gpu
#
self.evaluator = MultiWozEvaluator(self.reader)
if cfg.save_log and cfg.mode == 'train':
self.tb_writer = SummaryWriter(log_dir='./log')
else:
self.tb_writer = None
def get_optimizers(self):
"""
Setup the optimizer and the learning rate scheduler.
from transformers.Trainer
parameters from cfg: lr (1e-3); warmup_steps
"""
# Prepare optimizer and schedule (linear warmup and decay)
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in self.model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
cfg.weight_decay,
},
{
"params": [
p for n, p in self.model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0,
},
]
optimizer = AdamW(optimizer_grouped_parameters, lr=cfg.lr)
num_training_steps = self.reader.set_stats['train']['num_dials'] *\
cfg.epoch_num // (cfg.gradient_accumulation_steps*cfg.batch_size)
num_warmup_steps = cfg.warmup_steps if cfg.warmup_steps >= 0 else int(
num_training_steps * 0.2)
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=num_warmup_steps,
num_training_steps=num_training_steps)
return optimizer, scheduler
def log_first_inputs(self, inputs):
tokenizer = self.tokenizer
logging.info("**** Input Examples: ****")
for context in inputs['contexts'][:4]:
# ubar = tokenizer.convert_ids_to_tokens(context)
# ubar = tokenizer.convert_tokens_to_string(context)
# ubar = " ".join(ubar)
ubar = tokenizer.decode(context)
logging.info(ubar)
def add_torch_input(self, inputs):
# to tensor and to device
contexts_tensor = torch.from_numpy(inputs['contexts_np']).long()
contexts_tensor = contexts_tensor.to(self.device)
inputs['contexts_tensor'] = contexts_tensor
return inputs
def add_torch_input_eval(self, inputs):
# inputs: context
inputs['context_tensor'] = torch.tensor([inputs['context']
]).to(self.device)
return inputs
def calculate_loss_and_accuracy(self, outputs, labels):
# GPT2-chicahat/train.py
lm_logits = outputs[0]
shift_logits = lm_logits[..., :-1, :].contiguous()
shift_labels = labels[..., 1:].contiguous()
pad_id = cfg.pad_id
loss_fct = nn.CrossEntropyLoss(ignore_index=pad_id, reduction='sum')
loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)),
shift_labels.view(-1))
# avg loss
not_ignore = shift_labels.ne(pad_id)
num_targets = not_ignore.long().sum().item()
loss /= num_targets
return loss
def train_URURU(self):
"""
URURU
"""
all_batches = self.reader.get_batches('train')
# compute num_training_steps in get_batches()
optimizer, scheduler = self.get_optimizers()
# log info
set_stats = self.reader.set_stats['train']
logging.info("***** Running training *****")
logging.info(
" Num Training steps(one turn in a batch of dialogs) per epoch = %d",
set_stats['num_training_steps_per_epoch'])
logging.info(" Num Turns = %d", set_stats['num_turns'])
logging.info(" Num Dialogs = %d", set_stats['num_dials'])
logging.info(" Num Epochs = %d", cfg.epoch_num)
logging.info(" Batch size = %d", cfg.batch_size)
logging.info(" Gradient Accumulation steps = %d",
cfg.gradient_accumulation_steps)
logging.info(
" Total optimization steps = %d",
set_stats['num_training_steps_per_epoch'] * cfg.epoch_num //
cfg.gradient_accumulation_steps)
# tb writer
if self.tb_writer is not None:
self.tb_writer.add_text('cfg', json.dumps(cfg.__dict__, indent=2))
# self.tb_writer.add_hparams(self.args.to_sanitized_dict(), metric_dict={})
log_inputs = 2
global_step = 0
sw = time.time()
for epoch in range(cfg.epoch_num):
epoch_step = 0
tr_loss = 0.0
logging_loss = 0.0
btm = time.time()
oom_time = 0
self.model.zero_grad()
data_iterator = self.reader.get_data_iterator(all_batches)
for batch_idx, dial_batch in enumerate(data_iterator):
pv_batch = None
for turn_num, turn_batch in enumerate(dial_batch):
first_turn = (turn_num == 0)
inputs = self.reader.convert_batch_turn(
turn_batch, pv_batch, first_turn)
pv_batch = inputs['labels']
try: # avoid OOM
self.model.train()
if log_inputs > 0: # log inputs for the very first two turns
self.log_first_inputs(inputs)
log_inputs -= 1
# to tensor
inputs = self.add_torch_input(inputs)
# loss
outputs = self.model(inputs['contexts_tensor'])
# outputs = self.model(inputs['contexts_tensor']) # debugging with GPT2Model
loss = self.calculate_loss_and_accuracy(
outputs, labels=inputs['contexts_tensor'])
loss.backward()
tr_loss += loss.item()
torch.nn.utils.clip_grad_norm_(self.model.parameters(),
5.0)
epoch_step += 1
# step, wrt gradient_accumulation_steps, clip grad norm
if (epoch_step+1) % cfg.gradient_accumulation_steps == 0 or(
# end of an epoch
(epoch_step + \
1) == set_stats['num_training_steps_per_epoch']
):
optimizer.step()
scheduler.step()
optimizer.zero_grad()
# global_step: actual step the optimizer took
global_step += 1
logs = {} # for tb writer
# logging: loss, lr... after certain amount of steps
if cfg.report_interval > 0 and global_step % cfg.report_interval == 0:
loss_scalar = (tr_loss - logging_loss) / \
cfg.report_interval
logging_loss = tr_loss
logs['loss'] = loss_scalar
logging.info(
'Global step: {}, epoch step: {}, interval loss: {:.4f}'
.format(global_step, epoch_step,
loss_scalar))
# validate
# add to tensorboard...
if cfg.evaluate_during_training and loss_scalar < 10:
results = self.validate()
for k, v in results.items():
eval_key = "eval_{}".format(k)
logs[eval_key] = v
if self.tb_writer:
for k, v in logs.items():
self.tb_writer.add_scalar(
k, v, global_step)
# save model...
except RuntimeError as exception:
if "out of memory" in str(exception):
max_length = max(inputs['lengths'])
oom_time += 1
logging.info(
"WARNING: ran out of memory,times: {}, batch size: {}, max_len: {}"
.format(oom_time, cfg.batch_size, max_length))
if hasattr(torch.cuda, 'empty_cache'):
torch.cuda.empty_cache()
else:
logging.info(str(exception))
raise exception
logging.info(
'Train epoch time: {:.2f} min, epoch loss: {:.4f}'.format(
(time.time() - btm) / 60, tr_loss))
# save model after every epoch
# if epoch > 30 and tr_loss/epoch_step < 0.6:
self.save_model(epoch, tr_loss / epoch_step)
def train(self):
"""
UBARU
"""
all_batches = self.reader.get_batches('train')
# compute num_training_steps in get_batches()
optimizer, scheduler = self.get_optimizers()
# log info
set_stats = self.reader.set_stats['train']
logging.info("***** Running training *****")
logging.info(
" Num Training steps(one turn in a batch of dialogs) per epoch = %d",
set_stats['num_training_steps_per_epoch'])
logging.info(" Num Turns = %d", set_stats['num_turns'])
logging.info(" Num Dialogs = %d", set_stats['num_dials'])
logging.info(" Num Epochs = %d", cfg.epoch_num)
logging.info(" Batch size = %d", cfg.batch_size)
logging.info(" Gradient Accumulation steps = %d",
cfg.gradient_accumulation_steps)
logging.info(
" Total optimization steps = %d",
set_stats['num_dials'] * cfg.epoch_num //
(cfg.gradient_accumulation_steps * cfg.batch_size))
# tb writer
if self.tb_writer is not None:
self.tb_writer.add_text('cfg', json.dumps(cfg.__dict__, indent=2))
# self.tb_writer.add_hparams(self.args.to_sanitized_dict(), metric_dict={})
log_inputs = 2
global_step = 0
sw = time.time()
for epoch in range(cfg.epoch_num):
epoch_step = 0
tr_loss = 0.0
logging_loss = 0.0
btm = time.time()
oom_time = 0
self.model.zero_grad()
data_iterator = self.reader.get_nontranspose_data_iterator(
all_batches)
for batch_idx, dial_batch in enumerate(data_iterator):
inputs = self.reader.convert_batch_session(dial_batch)
try: # avoid OOM
self.model.train()
if log_inputs > 0: # log inputs for the very first two turns
self.log_first_inputs(inputs)
log_inputs -= 1
# to tensor
inputs = self.add_torch_input(inputs)
# loss
outputs = self.model(inputs['contexts_tensor'])
# outputs = self.model(inputs['contexts_tensor']) # debugging with GPT2Model
loss = self.calculate_loss_and_accuracy(
outputs, labels=inputs['contexts_tensor'])
loss.backward()
tr_loss += loss.item()
torch.nn.utils.clip_grad_norm_(self.model.parameters(),
5.0)
epoch_step += 1
# step, wrt gradient_accumulation_steps, clip grad norm
if (epoch_step+1) % cfg.gradient_accumulation_steps == 0 or(
# end of an epoch
(epoch_step + \
1) == set_stats['num_training_steps_per_epoch']
):
optimizer.step()
scheduler.step()
optimizer.zero_grad()
# global_step: actual step the optimizer took
global_step += 1
logs = {} # for tb writer
# logging: loss, lr... after certain amount of steps
if cfg.report_interval > 0 and global_step % cfg.report_interval == 0:
loss_scalar = (tr_loss - logging_loss) / \
cfg.report_interval
logging_loss = tr_loss
logs['loss'] = loss_scalar
logging.info(
'Global step: {}, epoch step: {}, interval loss: {:.4f}'
.format(global_step, epoch_step, loss_scalar))
# validate
# add to tensorboard...
if cfg.evaluate_during_training and loss_scalar < 10:
results = self.validate()
for k, v in results.items():
eval_key = "eval_{}".format(k)
logs[eval_key] = v
if self.tb_writer:
for k, v in logs.items():
self.tb_writer.add_scalar(
k, v, global_step)
# save model...
except RuntimeError as exception:
if "out of memory" in str(exception):
max_length = max(inputs['lengths'])
oom_time += 1
logging.info(
"WARNING: ran out of memory,times: {}, batch size: {}, max_len: {}"
.format(oom_time, cfg.batch_size, max_length))
if hasattr(torch.cuda, 'empty_cache'):
torch.cuda.empty_cache()
else:
logging.info(str(exception))
raise exception
logging.info(
'Train epoch time: {:.2f} min, epoch loss: {:.4f}'.format(
(time.time() - btm) / 60, tr_loss))
# save model after every epoch
# if epoch > 10 or tr_loss/epoch_step < 1:
self.save_model(epoch, tr_loss / epoch_step)
def save_model(self, epoch, loss):
save_path = os.path.join(
cfg.exp_path, 'epoch{}_trloss{:.2f}_gpt2'.format(epoch + 1, loss))
if not os.path.exists(save_path):
os.mkdir(save_path)
logging.info('Saving model checkpoint to %s', save_path)
# save gpt2
self.model.save_pretrained(save_path)
# save tokenizer
self.tokenizer.save_pretrained(save_path)
# save cfg
def validate_URURU(self, data='dev', do_test=False):
# predict one dialog/ one turn at a time
self.model.eval()
# all_batches = self.reader.get_batches('dev')
# data_iterator = self.reader.get_data_iterator(all_batches)
eval_data = self.reader.get_eval_data(data)
set_stats = self.reader.set_stats[data]
logging.info("***** Running Evaluation *****")
logging.info(" Num Turns = %d", set_stats['num_turns'])
# logging.info(" Num Dialogs = %d", set_stats['num_dials'])
# valid_losses = []
btm = time.time()
result_collection = {}
with torch.no_grad():
eval_pbar = eval_data
for dial_idx, dialog in enumerate(eval_pbar):
pv_turn = {}
for turn_idx, turn in enumerate(dialog):
first_turn = (turn_idx == 0)
inputs = self.reader.convert_turn_eval_URURU(
turn, pv_turn, first_turn)
inputs = self.add_torch_input_eval(inputs)
# fail to generate new tokens, if max_length not set
context_length = len(inputs['context'])
if cfg.use_true_curr_bspn: # generate act, response
max_len = 60
if not cfg.use_true_curr_aspn:
max_len = 80
outputs = self.model.generate(
input_ids=inputs['context_tensor'],
max_length=context_length + max_len,
temperature=0.7, # top_p=0.9, num_beams=4,
pad_token_id=self.tokenizer.eos_token_id,
eos_token_id=self.tokenizer.encode(['<eos_r>'])[0])
# no_repeat_ngram_size=4
# turn['generated'] = self.tokenizer.decode(outputs[0])
# resp_gen, need to trim previous context
generated = outputs[0].cpu().numpy().tolist()
generated = generated[context_length - 1:]
try:
decoded = self.decode_generated_act_resp(generated)
except ValueError as exception:
logging.info(str(exception))
logging.info(self.tokenizer.decode(generated))
decoded = {'resp': [], 'bspn': [], 'aspn': []}
else: # predict bspn, access db, then generate act and resp
outputs = self.model.generate(
input_ids=inputs['context_tensor'],
max_length=context_length + 60,
temperature=0.7, # top_p=0.9, num_beams=4,
pad_token_id=self.tokenizer.eos_token_id,
eos_token_id=self.tokenizer.encode(['<eos_b>'])[0])
generated_bs = outputs[0].cpu().numpy().tolist()
# generated_bs = generated_bs[context_length-1:]
bspn_gen = self.decode_generated_bspn(
generated_bs[context_length - 1:])
# check DB result
if cfg.use_true_db_pointer:
# db_result = self.reader.bspan_to_DBpointer(self.tokenizer.decode(turn['bspn']), turn['turn_domain'])
db = turn['db']
else:
db_result = self.reader.bspan_to_DBpointer(
self.tokenizer.decode(bspn_gen),
turn['turn_domain'])
db = self.tokenizer.convert_tokens_to_ids(
self.tokenizer.tokenize(
'<sos_db> ' + db_result +
' <eos_db>')) + self.tokenizer.encode(
['<sos_a>'])
inputs['context_tensor_db'] = torch.tensor([
inputs['context'][:-1] + bspn_gen + db
]).to(self.device)
context_length = len(inputs['context_tensor_db'][0])
outputs_db = self.model.generate(
input_ids=inputs['context_tensor_db'],
max_length=context_length + 80,
temperature=0.7, # top_p=0.9, num_beams=4,
pad_token_id=self.tokenizer.eos_token_id,
eos_token_id=self.tokenizer.encode(['<eos_r>'])[0])
generated_ar = outputs_db[0].cpu().numpy().tolist()
generated_ar = generated_ar[context_length - 1:]
try:
decoded = self.decode_generated_act_resp(
generated_ar)
decoded['bspn'] = bspn_gen
except ValueError as exception:
logging.info(str(exception))
logging.info(self.tokenizer.decode(generated_ar))
decoded = {'resp': [], 'bspn': [], 'aspn': []}
turn['resp_gen'] = decoded['resp']
turn['bspn_gen'] = turn[
'bspn'] if cfg.use_true_curr_bspn else decoded['bspn']
turn['aspn_gen'] = turn[
'aspn'] if cfg.use_true_curr_aspn else decoded['aspn']
turn['dspn_gen'] = turn['dspn']
# check DB results
# db_result = self.reader.bspan_to_DBpointer(self.tokenizer.decode(turn['bspn']), turn['turn_domain'])
# if db_result[0] == 1: # no match
# print('gt:', self.tokenizer.decode(turn['aspn']), ' |gen:', self.tokenizer.decode(decoded['aspn']))
# print('gen_resp: ', self.tokenizer.decode(decoded['resp']))
# print('gt_resp: ', self.tokenizer.decode(turn['resp']), '\n')
pv_turn['labels'] = inputs[
'labels'] # all true previous context
pv_turn['resp'] = turn[
'resp'] if cfg.use_true_prev_resp else decoded['resp']
# pv_turn['bspn'] = turn['bspn'] if cfg.use_true_prev_bspn else decoded['bspn']
# pv_turn['db'] = db
# pv_turn['aspn'] = turn['aspn'] if cfg.use_true_prev_aspn else decoded['aspn']
# pv_turn = inputs['labels']
result_collection.update(
self.reader.inverse_transpose_turn(dialog))
logging.info("inference time: {:.2f} min".format(
(time.time() - btm) / 60))
# score
btm = time.time()
results, _ = self.reader.wrap_result_lm(result_collection)
bleu, success, match = self.evaluator.validation_metric(results)
logging.info("Scoring time: {:.2f} min".format(
(time.time() - btm) / 60))
score = 0.5 * (success + match) + bleu
valid_loss = 130 - score
logging.info(
'validation [CTR] match: %2.2f success: %2.2f bleu: %2.2f score: %.2f'
% (match, success, bleu, score))
eval_results = {}
eval_results['bleu'] = bleu
eval_results['success'] = success
eval_results['match'] = match
return eval_results
def validate(self, data='dev', do_test=False):
# predict one dialog/ one turn at a time
self.model.eval()
# all_batches = self.reader.get_batches('dev')
# data_iterator = self.reader.get_data_iterator(all_batches)
eval_data = self.reader.get_eval_data(data)
set_stats = self.reader.set_stats[data]
logging.info("***** Running Evaluation *****")
logging.info(" Num Turns = %d", set_stats['num_turns'])
# logging.info(" Num Dialogs = %d", set_stats['num_dials'])
# valid_losses = []
btm = time.time()
result_collection = {}
with torch.no_grad():
for dial_idx, dialog in enumerate(eval_data):
pv_turn = {}
for turn_idx, turn in enumerate(dialog):
first_turn = (turn_idx == 0)
inputs = self.reader.convert_turn_eval(
turn, pv_turn, first_turn)
inputs = self.add_torch_input_eval(inputs)
# fail to generate new tokens, if max_length not set
context_length = len(inputs['context'])
if cfg.use_true_curr_bspn: # generate act, response
max_len = 60
if not cfg.use_true_curr_aspn:
max_len = 80
outputs = self.model.generate(
input_ids=inputs['context_tensor'],
max_length=context_length + max_len,
temperature=0.7, # top_p=0.9, num_beams=4,
pad_token_id=self.tokenizer.eos_token_id,
eos_token_id=self.tokenizer.encode(['<eos_r>'])[0])
# no_repeat_ngram_size=4
# turn['generated'] = self.tokenizer.decode(outputs[0])
# resp_gen, need to trim previous context
generated = outputs[0].cpu().numpy().tolist()
generated = generated[context_length - 1:]
try:
decoded = self.decode_generated_act_resp(generated)
except ValueError as exception:
logging.info(str(exception))
logging.info(self.tokenizer.decode(generated))
decoded = {'resp': [], 'bspn': [], 'aspn': []}
else: # predict bspn, access db, then generate act and resp
outputs = self.model.generate(
input_ids=inputs['context_tensor'],
max_length=context_length + 60,
temperature=0.7, # top_p=0.9, num_beams=4,
pad_token_id=self.tokenizer.eos_token_id,
eos_token_id=self.tokenizer.encode(['<eos_b>'])[0])
generated_bs = outputs[0].cpu().numpy().tolist()
# generated_bs = generated_bs[context_length-1:]
bspn_gen = self.decode_generated_bspn(
generated_bs[context_length - 1:])
# check DB result
if cfg.use_true_db_pointer:
# db_result = self.reader.bspan_to_DBpointer(self.tokenizer.decode(turn['bspn']), turn['turn_domain'])
db = turn['db']
else:
db_result = self.reader.bspan_to_DBpointer(
self.tokenizer.decode(bspn_gen),
turn['turn_domain'])
db = self.tokenizer.convert_tokens_to_ids(
self.tokenizer.tokenize(
'<sos_db> ' + db_result +
' <eos_db>')) + self.tokenizer.encode(
['<sos_a>'])
inputs['context_tensor_db'] = torch.tensor([
inputs['context'][:-1] + bspn_gen + db
]).to(self.device)
context_length = len(inputs['context_tensor_db'][0])
outputs_db = self.model.generate(
input_ids=inputs['context_tensor_db'],
max_length=context_length + 80,
temperature=0.7, # top_p=0.9, num_beams=4,
pad_token_id=self.tokenizer.eos_token_id,
eos_token_id=self.tokenizer.encode(['<eos_r>'])[0])
generated_ar = outputs_db[0].cpu().numpy().tolist()
generated_ar = generated_ar[context_length - 1:]
try:
decoded = self.decode_generated_act_resp(
generated_ar)
decoded['bspn'] = bspn_gen
except ValueError as exception:
logging.info(str(exception))
logging.info(self.tokenizer.decode(generated_ar))
decoded = {'resp': [], 'bspn': [], 'aspn': []}
turn['resp_gen'] = decoded['resp']
turn['bspn_gen'] = turn[
'bspn'] if cfg.use_true_curr_bspn else decoded['bspn']
turn['aspn_gen'] = turn[
'aspn'] if cfg.use_true_curr_aspn else decoded['aspn']
turn['dspn_gen'] = turn['dspn']
# check DB results
# db_result = self.reader.bspan_to_DBpointer(self.tokenizer.decode(turn['bspn']), turn['turn_domain'])
# if db_result[0] == 1: # no match
# print('gt:', self.tokenizer.decode(turn['aspn']), ' |gen:', self.tokenizer.decode(decoded['aspn']))
# print('gen_resp: ', self.tokenizer.decode(decoded['resp']))
# print('gt_resp: ', self.tokenizer.decode(turn['resp']), '\n')
pv_turn['labels'] = inputs[
'labels'] # all true previous context
pv_turn['resp'] = turn[
'resp'] if cfg.use_true_prev_resp else decoded['resp']
pv_turn['bspn'] = turn[
'bspn'] if cfg.use_true_prev_bspn else decoded['bspn']
pv_turn[
'db'] = turn['db'] if cfg.use_true_curr_bspn else db
pv_turn['aspn'] = turn[
'aspn'] if cfg.use_true_prev_aspn else decoded['aspn']
result_collection.update(
self.reader.inverse_transpose_turn(dialog))
logging.info("inference time: {:.2f} min".format(
(time.time() - btm) / 60))
# score
btm = time.time()
results, _ = self.reader.wrap_result_lm(result_collection)
bleu, success, match = self.evaluator.validation_metric(results)
logging.info("Scoring time: {:.2f} min".format(
(time.time() - btm) / 60))
score = 0.5 * (success + match) + bleu
valid_loss = 130 - score
logging.info(
'validation [CTR] match: %2.2f success: %2.2f bleu: %2.2f score: %.2f'
% (match, success, bleu, score))
eval_results = {}
eval_results['bleu'] = bleu
eval_results['success'] = success
eval_results['match'] = match
eval_results['score'] = score
eval_results[
'result'] = 'validation [CTR] match: %2.2f success: %2.2f bleu: %2.2f score: %.2f' % (
match, success, bleu, score)
model_setting, epoch_setting = cfg.eval_load_path.split(
'/')[1], cfg.eval_load_path.split('/')[2]
eval_on = '-'.join(cfg.exp_domains)
if data == 'test':
eval_on += '_test'
if not os.path.exists(cfg.log_path):
os.mkdir(cfg.log_path)
log_file_name = os.path.join(cfg.log_path,
model_setting + '-' + eval_on + '.json')
if os.path.exists(log_file_name):
eval_to_json = json.load(open(log_file_name, 'r'))
eval_to_json[epoch_setting] = eval_results
json.dump(eval_to_json, open(log_file_name, 'w'), indent=2)
else:
eval_to_json = {}
eval_to_json[epoch_setting] = eval_results
json.dump(eval_to_json, open(log_file_name, 'w'), indent=2)
logging.info('update eval results to {}'.format(log_file_name))
return eval_results
def decode_generated_act_resp(self, generated):
"""
decode generated
return decoded['resp'] ('bspn', 'aspn')
"""
decoded = {}
eos_a_id = self.tokenizer.encode(['<eos_a>'])[0]
eos_r_id = self.tokenizer.encode(['<eos_r>'])[0]
eos_b_id = self.tokenizer.encode(['<eos_b>'])[0]
# eos_r may not exists if gpt2 generated repetitive words.
if eos_r_id in generated:
eos_r_idx = generated.index(eos_r_id)
else:
eos_r_idx = len(generated) - 1
logging.info('eos_r not in generated: ' +
self.tokenizer.decode(generated))
# eos_r_idx = generated.index(eos_r_id) if eos_r_id in generated else len(generated)-1
if cfg.use_true_curr_aspn: # only predict resp
decoded['resp'] = generated[:eos_r_idx + 1]
else: # predicted aspn, resp
eos_a_idx = generated.index(eos_a_id)
decoded['aspn'] = generated[:eos_a_idx + 1]
decoded['resp'] = generated[eos_a_idx + 1:eos_r_idx + 1]
# if cfg.use_true_curr_bspn:
# else: # predict bspn aspn resp
# eos_b_idx = generated.index(eos_b_id)
# eos_a_idx = generated.index(eos_a_id)
# decoded['bspn'] = generated[: eos_b_idx+1]
# decoded['aspn'] = generated[eos_b_idx+1: eos_a_idx+1]
# decoded['resp'] = generated[eos_a_idx+1: eos_r_idx+1]
return decoded
def decode_generated_bspn(self, generated):
eos_b_id = self.tokenizer.encode(['<eos_b>'])[0]
if eos_b_id in generated:
eos_b_idx = generated.index(eos_b_id)
else:
eos_b_idx = len(generated) - 1
return generated[:eos_b_idx + 1]
class Model(object):
def __init__(self):
self.reader = MultiWozReader()
if len(cfg.cuda_device) == 1:
self.m = DAMD(self.reader)
else:
m = DAMD(self.reader)
self.m = torch.nn.DataParallel(m, device_ids=cfg.cuda_device)
# print(self.m.module)
self.evaluator = MultiWozEvaluator(self.reader) # evaluator class
if cfg.cuda: self.m = self.m.cuda() #cfg.cuda_device[0]
self.optim = Adam(lr=cfg.lr,
params=filter(lambda x: x.requires_grad,
self.m.parameters()),
weight_decay=5e-5)
self.base_epoch = -1
if cfg.limit_bspn_vocab:
self.reader.bspn_masks_tensor = {}
for key, values in self.reader.bspn_masks.items():
v_ = cuda_(torch.Tensor(values).long())
self.reader.bspn_masks_tensor[key] = v_
if cfg.limit_aspn_vocab:
self.reader.aspn_masks_tensor = {}
for key, values in self.reader.aspn_masks.items():
v_ = cuda_(torch.Tensor(values).long())
self.reader.aspn_masks_tensor[key] = v_
def add_torch_input(self, inputs, mode='train', first_turn=False):
need_onehot = [
'user', 'usdx', 'bspn', 'aspn', 'pv_resp', 'pv_bspn', 'pv_aspn',
'dspn', 'pv_dspn', 'bsdx', 'pv_bsdx'
]
inputs['db'] = cuda_(torch.from_numpy(inputs['db_np']).float())
for item in ['user', 'usdx', 'resp', 'bspn', 'aspn', 'bsdx', 'dspn']:
if not cfg.enable_aspn and item == 'aspn':
continue
if not cfg.enable_bspn and item == 'bspn':
continue
if not cfg.enable_dspn and item == 'dspn':
continue
inputs[item] = cuda_(
torch.from_numpy(
inputs[item + '_unk_np']).long()) # replace oov to <unk>
if item in ['user', 'usdx', 'resp', 'bspn']:
inputs[item + '_nounk'] = cuda_(
torch.from_numpy(
inputs[item +
'_np']).long()) # don't replace oov to <unk>
else:
inputs[item + '_nounk'] = inputs[item]
# print(item, inputs[item].size())
if item in ['resp', 'bspn', 'aspn', 'bsdx', 'dspn']:
if 'pv_' + item + '_unk_np' not in inputs:
continue
inputs['pv_' + item] = cuda_(
torch.from_numpy(inputs['pv_' + item + '_unk_np']).long())
if item in ['user', 'usdx', 'bspn']:
inputs['pv_' + item + '_nounk'] = cuda_(
torch.from_numpy(inputs['pv_' + item + '_np']).long())
inputs[item + '_4loss'] = self.index_for_loss(item, inputs)
else:
inputs['pv_' + item + '_nounk'] = inputs['pv_' + item]
inputs[item + '_4loss'] = inputs[item]
if 'pv_' + item in need_onehot:
inputs['pv_' + item + '_onehot'] = get_one_hot_input(
inputs['pv_' + item + '_unk_np'])
if item in need_onehot:
inputs[item + '_onehot'] = get_one_hot_input(inputs[item +
'_unk_np'])
if cfg.multi_acts_training and 'aspn_aug_unk_np' in inputs:
inputs['aspn_aug'] = cuda_(
torch.from_numpy(inputs['aspn_aug_unk_np']).long())
inputs['aspn_aug_4loss'] = inputs['aspn_aug']
return inputs
def index_for_loss(self, item, inputs):
raw_labels = inputs[item + '_np']
if item == 'bspn':
copy_sources = [
inputs['user_np'], inputs['pv_resp_np'], inputs['pv_bspn_np']
]
elif item == 'bsdx':
copy_sources = [
inputs['usdx_np'], inputs['pv_resp_np'], inputs['pv_bsdx_np']
]
elif item == 'aspn':
copy_sources = []
if cfg.use_pvaspn:
copy_sources.append(inputs['pv_aspn_np'])
if cfg.enable_bspn:
copy_sources.append(inputs[cfg.bspn_mode + '_np'])
elif item == 'dspn':
copy_sources = [inputs['pv_dspn_np']]
elif item == 'resp':
copy_sources = [inputs['usdx_np']]
if cfg.enable_bspn:
copy_sources.append(inputs[cfg.bspn_mode + '_np'])
if cfg.enable_aspn:
copy_sources.append(inputs['aspn_np'])
else:
return
new_labels = np.copy(raw_labels)
if copy_sources:
bidx, tidx = np.where(raw_labels >= self.reader.vocab_size)
copy_sources = np.concatenate(copy_sources, axis=1)
for b in bidx:
for t in tidx:
oov_idx = raw_labels[b, t]
if len(np.where(copy_sources[b, :] == oov_idx)[0]) == 0:
new_labels[b, t] = 2
return cuda_(torch.from_numpy(new_labels).long())
def train(self):
lr = cfg.lr
prev_min_loss, early_stop_count = 1 << 30, cfg.early_stop_count
weight_decay_count = cfg.weight_decay_count
train_time = 0
sw = time.time()
for epoch in range(cfg.epoch_num):
if epoch <= self.base_epoch:
continue
self.training_adjust(epoch)
sup_loss = 0
sup_cnt = 0
optim = self.optim
# data_iterator generatation size: (batch num, turn num, batch size)
btm = time.time()
data_iterator = self.reader.get_batches('train')
for iter_num, dial_batch in enumerate(data_iterator):
hidden_states = {}
py_prev = {
'pv_resp': None,
'pv_bspn': None,
'pv_aspn': None,
'pv_dspn': None,
'pv_bsdx': None
}
bgt = time.time()
for turn_num, turn_batch in enumerate(dial_batch):
# print('turn %d'%turn_num)
# print(len(turn_batch['dial_id']))
optim.zero_grad()
first_turn = (turn_num == 0)
inputs = self.reader.convert_batch(turn_batch,
py_prev,
first_turn=first_turn)
inputs = self.add_torch_input(inputs,
first_turn=first_turn)
# total_loss, losses, hidden_states = self.m(inputs, hidden_states, first_turn, mode='train')
total_loss, losses = self.m(inputs,
hidden_states,
first_turn,
mode='train')
# print('forward completed')
py_prev['pv_resp'] = turn_batch['resp']
if cfg.enable_bspn:
py_prev['pv_bspn'] = turn_batch['bspn']
py_prev['pv_bsdx'] = turn_batch['bsdx']
if cfg.enable_aspn:
py_prev['pv_aspn'] = turn_batch['aspn']
if cfg.enable_dspn:
py_prev['pv_dspn'] = turn_batch['dspn']
total_loss = total_loss.mean()
# print('forward time:%f'%(time.time()-test_begin))
# test_begin = time.time()
total_loss.backward(retain_graph=False)
# total_loss.backward(retain_graph=turn_num != len(dial_batch) - 1)
# print('backward time:%f'%(time.time()-test_begin))
grad = torch.nn.utils.clip_grad_norm_(
self.m.parameters(), 5.0)
optim.step()
sup_loss += float(total_loss)
sup_cnt += 1
torch.cuda.empty_cache()
if (iter_num + 1) % cfg.report_interval == 0:
logging.info(
'iter:{} [total|bspn|aspn|resp|ptr|gate] loss: {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} grad:{:.2f} time: {:.1f} turn:{} '.format(iter_num+1, \
float(total_loss),float(losses['bspn']),float(losses['aspn']),float(losses['resp']), float(losses["trade_ptr"]), \
float(losses["trade_gating"]),grad, time.time()-btm, turn_num+1))
if cfg.enable_dst and cfg.bspn_mode == 'bsdx':
logging.info('bspn-dst:{:.3f}'.format(
float(losses['bspn'])))
if cfg.multi_acts_training:
logging.info('aspn-aug:{:.3f}'.format(
float(losses['aspn_aug'])))
# btm = time.time()
# if (iter_num+1)%40==0:
# print('validation checking ... ')
# valid_sup_loss, valid_unsup_loss = self.validate(do_test=False)
# logging.info('validation loss in epoch %d sup:%f unsup:%f' % (epoch, valid_sup_loss, valid_unsup_loss))
epoch_sup_loss = sup_loss / (sup_cnt + 1e-8)
# do_test = True if (epoch+1)%5==0 else False
do_test = False
valid_loss = self.validate(do_test=do_test)
logging.info(
'epoch: %d, train loss: %.3f, valid loss: %.3f, total time: %.1fmin'
% (epoch + 1, epoch_sup_loss, valid_loss,
(time.time() - sw) / 60))
# self.save_model(epoch)
if valid_loss <= prev_min_loss:
early_stop_count = cfg.early_stop_count
weight_decay_count = cfg.weight_decay_count
prev_min_loss = valid_loss
self.save_model(epoch)
else:
early_stop_count -= 1
weight_decay_count -= 1
logging.info('epoch: %d early stop countdown %d' %
(epoch + 1, early_stop_count))
if not early_stop_count:
self.load_model()
print('result preview...')
file_handler = logging.FileHandler(
os.path.join(cfg.exp_path,
'eval_log%s.json' % cfg.seed))
logging.getLogger('').addHandler(file_handler)
logging.info(str(cfg))
self.eval()
return
if not weight_decay_count:
lr *= cfg.lr_decay
self.optim = Adam(lr=lr,
params=filter(lambda x: x.requires_grad,
self.m.parameters()),
weight_decay=5e-5)
weight_decay_count = cfg.weight_decay_count
logging.info('learning rate decay, learning rate: %f' %
(lr))
self.load_model()
print('result preview...')
file_handler = logging.FileHandler(
os.path.join(cfg.exp_path, 'eval_log%s.json' % cfg.seed))
logging.getLogger('').addHandler(file_handler)
logging.info(str(cfg))
self.eval()
def validate(self, data='dev', do_test=False):
self.m.eval()
valid_loss, count = 0, 0
data_iterator = self.reader.get_batches(data)
result_collection = {}
for batch_num, dial_batch in enumerate(data_iterator):
hidden_states = {}
py_prev = {
'pv_resp': None,
'pv_bspn': None,
'pv_aspn': None,
'pv_dspn': None,
'pv_bsdx': None
}
for turn_num, turn_batch in enumerate(dial_batch):
first_turn = (turn_num == 0)
inputs = self.reader.convert_batch(turn_batch,
py_prev,
first_turn=first_turn)
inputs = self.add_torch_input(inputs, first_turn=first_turn)
# total_loss, losses, hidden_states = self.m(inputs, hidden_states, first_turn, mode='train')
if cfg.valid_loss not in ['score', 'match', 'success', 'bleu']:
total_loss, losses = self.m(inputs,
hidden_states,
first_turn,
mode='train')
py_prev['pv_resp'] = turn_batch['resp']
if cfg.enable_bspn:
py_prev['pv_bspn'] = turn_batch['bspn']
py_prev['pv_bsdx'] = turn_batch['bsdx']
if cfg.enable_aspn:
py_prev['pv_aspn'] = turn_batch['aspn']
if cfg.enable_dspn:
py_prev['pv_dspn'] = turn_batch['dspn']
if cfg.valid_loss == 'total_loss':
valid_loss += float(total_loss)
elif cfg.valid_loss == 'bspn_loss':
valid_loss += float(losses[cfg.bspn_mode])
elif cfg.valid_loss == 'aspn_loss':
valid_loss += float(losses['aspn'])
elif cfg.valid_loss == 'resp_loss':
valid_loss += float(losses['reps'])
else:
raise ValueError('Invalid validation loss type!')
else:
decoded = self.m(inputs,
hidden_states,
first_turn,
mode='test')
turn_batch['resp_gen'] = decoded['resp']
if cfg.bspn_mode == 'bspn' or cfg.enable_dst:
turn_batch['bspn_gen'] = decoded['bspn']
py_prev['pv_resp'] = turn_batch[
'resp'] if cfg.use_true_pv_resp else decoded['resp']
if cfg.enable_bspn:
py_prev['pv_' + cfg.bspn_mode] = turn_batch[
cfg.
bspn_mode] if cfg.use_true_prev_bspn else decoded[
cfg.bspn_mode]
py_prev['pv_bspn'] = turn_batch[
'bspn'] if cfg.use_true_prev_bspn or 'bspn' not in decoded else decoded[
'bspn']
if cfg.enable_aspn:
py_prev['pv_aspn'] = turn_batch[
'aspn'] if cfg.use_true_prev_aspn else decoded[
'aspn']
if cfg.enable_dspn:
py_prev['pv_dspn'] = turn_batch[
'dspn'] if cfg.use_true_prev_dspn else decoded[
'dspn']
# TRADE
if cfg.enable_trade:
turn_batch["trade_ptr"] = decoded["trade_ptr"]
turn_batch["trade_gate"] = decoded["trade_gate"]
count += 1
torch.cuda.empty_cache()
if cfg.valid_loss in ['score', 'match', 'success', 'bleu']:
result_collection.update(
self.reader.inverse_transpose_batch(dial_batch))
if cfg.valid_loss not in ['score', 'match', 'success', 'bleu']:
valid_loss /= (count + 1e-8)
else:
results, _ = self.reader.wrap_result(
result_collection) # decode to sentence
bleu, success, match = self.evaluator.validation_metric(results)
score = 0.5 * (success + match) + bleu
valid_loss = 200 - score
logging.info(
'validation [CTR] match: %2.1f success: %2.1f bleu: %2.1f' %
(match, success, bleu))
self.m.train()
if do_test:
print('result preview...')
self.eval()
return valid_loss
def eval(self, data='test'):
self.m.eval()
self.reader.result_file = None
result_collection = {}
data_iterator = self.reader.get_batches(data)
for batch_num, dial_batch in tqdm.tqdm(enumerate(data_iterator)):
# quit()
# if batch_num > 0:
# continue
hidden_states = {}
py_prev = {
'pv_resp': None,
'pv_bspn': None,
'pv_aspn': None,
'pv_dspn': None,
'pv_bsdx': None
}
print('batch_size:', len(dial_batch[0]['resp']))
for turn_num, turn_batch in enumerate(dial_batch):
# print('turn %d'%turn_num)
# if turn_num!=0 and turn_num<4:
# continue
first_turn = (turn_num == 0)
inputs = self.reader.convert_batch(turn_batch,
py_prev,
first_turn=first_turn)
inputs = self.add_torch_input(inputs, first_turn=first_turn)
decoded = self.m(inputs,
hidden_states,
first_turn,
mode='test')
turn_batch['resp_gen'] = decoded['resp']
if cfg.bspn_mode == 'bsdx':
turn_batch['bsdx_gen'] = decoded[
'bsdx'] if cfg.enable_bspn else [[0]] * len(
decoded['resp'])
if cfg.bspn_mode == 'bspn' or cfg.enable_dst:
turn_batch['bspn_gen'] = decoded[
'bspn'] if cfg.enable_bspn else [[0]] * len(
decoded['resp'])
turn_batch['aspn_gen'] = decoded[
'aspn'] if cfg.enable_aspn else [[0]] * len(
decoded['resp'])
turn_batch['dspn_gen'] = decoded[
'dspn'] if cfg.enable_dspn else [[0]] * len(
decoded['resp'])
if self.reader.multi_acts_record is not None:
turn_batch['multi_act_gen'] = self.reader.multi_acts_record
if cfg.record_mode:
turn_batch['multi_act'] = self.reader.aspn_collect
turn_batch['multi_resp'] = self.reader.resp_collect
# print(turn_batch['user'])
# print('user:'******'user'][0] , eos='<eos_u>', indicate_oov=True))
# print('resp:', self.reader.vocab.sentence_decode(decoded['resp'][0] , eos='<eos_r>', indicate_oov=True))
# print('bspn:', self.reader.vocab.sentence_decode(decoded['bspn'][0] , eos='<eos_b>', indicate_oov=True))
# for b in range(len(decoded['resp'])):
# for i in range(5):
# print('aspn:', self.reader.vocab.sentence_decode(decoded['aspn'][i][b] , eos='<eos_a>', indicate_oov=True))
py_prev['pv_resp'] = turn_batch[
'resp'] if cfg.use_true_pv_resp else decoded['resp']
if cfg.enable_bspn:
py_prev['pv_' + cfg.bspn_mode] = turn_batch[
cfg.bspn_mode] if cfg.use_true_prev_bspn else decoded[
cfg.bspn_mode]
py_prev['pv_bspn'] = turn_batch[
'bspn'] if cfg.use_true_prev_bspn or 'bspn' not in decoded else decoded[
'bspn']
if cfg.enable_aspn:
py_prev['pv_aspn'] = turn_batch[
'aspn'] if cfg.use_true_prev_aspn else decoded['aspn']
if cfg.enable_dspn:
py_prev['pv_dspn'] = turn_batch[
'dspn'] if cfg.use_true_prev_dspn else decoded['dspn']
# TRADE
if cfg.enable_trade:
turn_batch["trade_ptr"] = decoded["trade_ptr"]
turn_batch["trade_gate"] = decoded["trade_gate"]
torch.cuda.empty_cache()
# prev_z = turn_batch['bspan']
# print('test iter %d'%(batch_num+1))
result_collection.update(
self.reader.inverse_transpose_batch(dial_batch))
# self.reader.result_file.close()
if cfg.record_mode:
self.reader.record_utterance(result_collection)
quit()
results, field = self.reader.wrap_result(result_collection)
self.reader.save_result('w', results, field)
metric_results = self.evaluator.run_metrics(results)
metric_field = list(metric_results[0].keys())
req_slots_acc = metric_results[0]['req_slots_acc']
info_slots_acc = metric_results[0]['info_slots_acc']
self.reader.save_result('w',
metric_results,
metric_field,
write_title='EVALUATION RESULTS:')
self.reader.save_result('a', [info_slots_acc],
list(info_slots_acc.keys()),
write_title='INFORM ACCURACY OF EACH SLOTS:')
self.reader.save_result('a', [req_slots_acc],
list(req_slots_acc.keys()),
write_title='REQUEST SUCCESS RESULTS:')
self.reader.save_result('a',
results,
field +
['wrong_domain', 'wrong_act', 'wrong_inform'],
write_title='DECODED RESULTS:')
self.reader.save_result_report(metric_results)
# self.reader.metric_record(metric_results)
self.m.train()
return None
def save_model(self, epoch, path=None, critical=False):
if not cfg.save_log:
return
if not path:
path = cfg.model_path
if critical:
path += '.final'
all_state = {
'lstd': self.m.state_dict(),
'config': cfg.__dict__,
'epoch': epoch
}
torch.save(all_state, path)
logging.info('Model saved')
def load_model(self, path=None):
if not path:
path = cfg.model_path
all_state = torch.load(path, map_location='cpu')
self.m.load_state_dict(all_state['lstd'])
self.base_epoch = all_state.get('epoch', 0)
logging.info('Model loaded')
def training_adjust(self, epoch):
return
def freeze_module(self, module):
for param in module.parameters():
param.requires_grad = False
def unfreeze_module(self, module):
for param in module.parameters():
param.requires_grad = True
def load_glove_embedding(self, freeze=False):
if not cfg.multi_gpu:
initial_arr = self.m.embedding.weight.data.cpu().numpy()
emb = torch.from_numpy(
utils.get_glove_matrix(cfg.glove_path, self.reader.vocab,
initial_arr))
self.m.embedding.weight.data.copy_(emb)
else:
initial_arr = self.m.module.embedding.weight.data.cpu().numpy()
emb = torch.from_numpy(
utils.get_glove_matrix(cfg.glove_path, self.reader.vocab,
initial_arr))
self.m.module.embedding.weight.data.copy_(emb)
def count_params(self):
module_parameters = filter(lambda p: p.requires_grad,
self.m.parameters())
param_cnt = int(sum([np.prod(p.size()) for p in module_parameters]))
print('total trainable params: %d' % param_cnt)
return param_cnt
def dialog_turn_state_analysis(mode='train'):
data_path = 'data/multi-woz-processed/data_for_damd.json'
conv_data = 'data/multi-woz/annotated_user_da_with_span_full.json'
archive = zipfile.ZipFile(conv_data + '.zip', 'r')
convlab_data = json.loads(archive.open(conv_data.split('/')[-1], 'r').read().lower())
reader = MultiWozReader()
data = json.loads(open(data_path, 'r', encoding='utf-8').read().lower())
turn_state_record, turn_state_count, golden_acts = {}, {}, {}
act_state_collect = []
act_state_detail = {}
state_valid_acts = {}
dial_count = 0
turn_count = 0
for fn, dial in data.items():
dial_count += 1
state_valid_acts[fn] = {}
for turn_no, turn in enumerate(dial['log']):
turn_state = {}
turn_domain = turn['turn_domain'].split()
cons_delex = turn['cons_delex'].split()
sys_act = turn['sys_act']
usr_act = convlab_data[fn]['log'][turn_no * 2]['dialog_act']
db_ptr = [int(i) for i in turn['pointer'].split(',')]
match = turn['match']
if len(turn_domain) != 1 or turn_domain[0] == '[general]' or not sys_act:
continue
state_valid_acts[fn][turn_no] = {}
turn_count += 1
slot_mentioned = []
for idx, tk in enumerate(cons_delex[:-1]):
if tk in turn_domain:
i = idx+1
while i < len(cons_delex):
if '[' not in cons_delex[i]:
slot_mentioned.append(cons_delex[i])
else:
break
i = i+1
slot_mentioned.sort()
# turn_state['slot_mentioned'] = len(slot_mentioned)
turn_state['domain'] = turn_domain
turn_state['slot_mentioned'] = slot_mentioned
if match == '':
turn_state['match']=''
elif match == '0':
turn_state['match']='0'
elif match == '1':
turn_state['match'] = '1'
elif match == '2' or match == '3':
turn_state['match'] = '2-3'
else:
turn_state['match']='>3'
if db_ptr[-2:] == [0,0]:
turn_state['book'] = ''
elif db_ptr[-2:] == [1,0]:
turn_state['book'] = 'no'
else:
turn_state['book'] = 'yes'
turn_state['usract'] = []
for act in usr_act:
d, a = act.split('-')
if a not in turn_state['usract']:
slot_list = []
if a == 'request':
for slot_value in usr_act[act]:
slot = slot_value[0]
if slot == 'none':
continue
elif slot not in slot_list:
slot = ontology.da_abbr_to_slot_name.get(slot, slot)
slot_list.append(slot)
if not slot_list:
turn_state['usract'].append(a)
else:
slot_list.sort()
turn_state['usract'].append(a+'('+','.join(slot_list)+')')
turn_state['usract'].sort()
turn_state_str = ''
for k,v in turn_state.items():
if isinstance(v, list):
v_ = ','.join(v)
elif isinstance(v, int):
v_ = str(v)
else:
v_ = v
turn_state_str += '%s(%s);'%(k, v_)
turn_state_str = turn_state_str[:-1]
state_valid_acts[fn][turn_no]['usdx'] = turn['user_delex']
state_valid_acts[fn][turn_no]['state'] = turn_state_str
if sys_act not in act_state_detail:
act_state_detail[sys_act] = 1
act_list = reader.aspan_to_act_list(sys_act)
act_state = {'domain': {}, 'general': {}}
for act in act_list:
d, a, s = act.split('-')
if d == 'general':
act_state['general'][a] = ''
else:
if a not in act_state['domain']:
if s != 'none':
act_state['domain'][a] = ''
else:
act_state['domain'][a] = ''
else:
act_state['domain'][a] = ''
no_order_act = {}
for a in act_list:
no_order_act[a] = 1
act_state_str = ''
for k,v in act_state.items():
if isinstance(v, dict):
v_ = ''
for kk, vv in v.items():
v_ += kk+'(%s),'%str(vv)
if v_.endswith(','):
v_ = v_[:-1]
elif isinstance(v, int):
v_ = str(v)
else:
v_ = v
if v_ != '':
act_state_str += '%s(%s);'%(k, v_)
act_state_str = act_state_str[:-1]
state_valid_acts[fn][turn_no]['gold'] = {}
state_valid_acts[fn][turn_no]['gold'][act_state_str] = {}
state_valid_acts[fn][turn_no]['gold'][act_state_str]['resp'] = turn['resp']
state_valid_acts[fn][turn_no]['gold'][act_state_str]['act'] = sys_act
if mode == 'test' and fn not in reader.test_files:
continue
if mode == 'train' and fn in reader.test_files:
continue
if act_state not in act_state_collect:
act_state_collect.append(act_state)
new_state = True if turn_state_str not in turn_state_record else False
raw_sys_rec = fn+'-'+str(turn_no)+':'+sys_act
if new_state:
turn_state_record[turn_state_str] = {act_state_str: {'num': 1, 'raw_acts': [raw_sys_rec], 'no_order_act': [no_order_act],
'user': [turn['user']], 'resp': [turn['resp']]}}
golden_acts[turn_state_str] = {'act_span': raw_sys_rec, 'no_order_act': no_order_act}
turn_state_count[turn_state_str] = 1
else:
turn_state_count[turn_state_str] += 1
if act_state_str in turn_state_record[turn_state_str]:
if no_order_act == golden_acts[turn_state_str]['no_order_act']:
continue
if no_order_act in turn_state_record[turn_state_str][act_state_str]['no_order_act']:
continue
turn_state_record[turn_state_str][act_state_str]['num'] +=1
turn_state_record[turn_state_str][act_state_str]['raw_acts'].append(raw_sys_rec)
turn_state_record[turn_state_str][act_state_str]['user'].append(turn['user'])
turn_state_record[turn_state_str][act_state_str]['resp'].append(turn['resp'])
turn_state_record[turn_state_str][act_state_str]['no_order_act'].append(no_order_act)
else:
turn_state_record[turn_state_str][act_state_str] = {'num': 1, 'raw_acts': [raw_sys_rec], 'no_order_act': [no_order_act],
'user': [turn['user']], 'resp': [turn['resp']]}
for state, acts in turn_state_record.items():
turn_state_record[state] = OrderedDict(sorted(acts.items(), key=lambda i:i[1]['num'], reverse=True))
# print(mode)
print('dialog count:', dial_count, 'turn count: ',turn_count)
print('state count:', len(turn_state_record))
print('raw act span count:', len(act_state_detail))
print('act state count:', len(act_state_collect))
for fn, dial in data.items():
if fn in reader.dev_files or fn in reader.test_files:
continue
dial_count += 1
for turn_no, turn in enumerate(dial['log']):
if turn_no not in state_valid_acts[fn]:
continue
state = state_valid_acts[fn][turn_no]['state']
gold_act_type = list(state_valid_acts[fn][turn_no]['gold'].keys())[0]
state_valid_acts[fn][turn_no]['other'] = {}
if state in turn_state_record:
for act_type in turn_state_record[state]:
if act_type == gold_act_type:
continue
state_valid_acts[fn][turn_no]['other'][act_type] = []
for idx, a in enumerate(turn_state_record[state][act_type]['raw_acts']):
m = {'act': a}
m['resp'] = turn_state_record[state][act_type]['resp'][idx]
state_valid_acts[fn][turn_no]['other'][act_type].append(m)
# sub_state_valid_acts = {}
# count = 0
# for fn, dial in state_valid_acts.items():
# if 'mul' in fn and fn not in reader.test_files and count<=100:
# sub_state_valid_acts[fn] = dial
# count += 1
# if count >100:
# break
# with open('data/multi-woz-processed/example_multi_act_dialogs.json', 'w') as f:
# json.dump(sub_state_valid_acts, f, indent=2)
idx_save = {}
act_span_save = {}
hist = []
for fn, dial in state_valid_acts.items():
if fn in reader.dev_files or fn in reader.test_files:
continue
act_span_save[fn] = {}
idx_save[fn] = {}
for turn_num, turn in dial.items():
act_span_save[fn][turn_num] = {}
idx_save[fn][turn_num] = []
for act_type, acts in turn['other'].items():
hist.append(len(acts)+1)
act_span_save[fn][turn_num][act_type] = [a['act'].split(':')[1] for a in acts]
idx_save[fn][turn_num].append([a['act'].split(':')[0] for a in acts])
with open('data/multi-woz-processed/multi_act_mapping_%s.json'%mode, 'w') as f:
json.dump(act_span_save, f, indent=2)