def __init__(self):
root_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json')
voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json')
self.vector = CrossWozVector(sys_da_voc_json=voc_file, usr_da_voc_json=voc_opp_file)
processed_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'processed_data')
if os.path.exists(processed_dir):
print('Load processed data file')
self._load_data(processed_dir)
else:
print('Start preprocessing the dataset')
self._build_data(root_dir, processed_dir)
def __init__(self,
archive_file=DEFAULT_ARCHIVE_FILE,
model_file='https://convlab.blob.core.windows.net/convlab-2/mle_policy_crosswoz.zip'):
root_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json'), 'r') as f:
cfg = json.load(f)
voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json')
voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json')
self.vector = CrossWozVector(sys_da_voc_json=voc_file, usr_da_voc_json=voc_opp_file)
self.policy = MultiDiscretePolicy(self.vector.state_dim, cfg['h_dim'], self.vector.sys_da_dim).to(device=DEVICE)
if not os.path.isfile(archive_file):
if not model_file:
raise Exception("No model for MLE Policy is specified!")
archive_file = cached_path(model_file)
model_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'save')
if not os.path.exists(model_dir):
os.mkdir(model_dir)
if not os.path.exists(os.path.join(model_dir, 'best_mle.pol.mdl')):
archive = zipfile.ZipFile(archive_file, 'r')
archive.extractall(model_dir)
self.load_from_pretrained(archive_file, model_file, cfg['load'])
def __init__(self, manager, cfg):
self.data_train = manager.create_dataset('train', cfg['batchsz'])
self.data_valid = manager.create_dataset('val', cfg['batchsz'])
self.data_test = manager.create_dataset('test', cfg['batchsz'])
self.save_dir = cfg['save_dir']
self.print_per_batch = cfg['print_per_batch']
self.save_per_epoch = cfg['save_per_epoch']
voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json')
voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json')
vector = CrossWozVector(voc_file, voc_opp_file)
self.policy = MultiDiscretePolicy(vector.state_dim, cfg['h_dim'],
vector.sys_da_dim).to(device=DEVICE)
self.policy.eval()
self.policy_optim = torch.optim.Adam(self.policy.parameters(),
lr=cfg['lr'])
self.multi_entropy_loss = nn.MultiLabelSoftMarginLoss()
class PolicyDataLoaderCrossWoz():
def __init__(self):
root_dir = os.path.dirname(
os.path.dirname(
os.path.dirname(
os.path.dirname(os.path.dirname(
os.path.abspath(__file__))))))
voc_file = os.path.join(root_dir, 'data/crosswoz/sys_da_voc.json')
voc_opp_file = os.path.join(root_dir, 'data/crosswoz/usr_da_voc.json')
self.vector = CrossWozVector(sys_da_voc_json=voc_file,
usr_da_voc_json=voc_opp_file)
processed_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'processed_data')
if os.path.exists(processed_dir):
print('Load processed data file')
self._load_data(processed_dir)
else:
print('Start preprocessing the dataset')
self._build_data(root_dir, processed_dir)
def _build_data(self, root_dir, processed_dir):
raw_data = {}
for part in ['train', 'val', 'test']:
archive = zipfile.ZipFile(
os.path.join(root_dir,
'data/crosswoz/{}.json.zip'.format(part)), 'r')
with archive.open('{}.json'.format(part), 'r') as f:
raw_data[part] = json.load(f)
self.data = {}
# for cur domain update
dst = RuleDST()
for part in ['train', 'val', 'test']:
self.data[part] = []
for key in raw_data[part]:
sess = raw_data[part][key]['messages']
dst.init_session()
for i, turn in enumerate(sess):
if turn['role'] == 'usr':
dst.state['user_action'] = turn['dialog_act']
dst.update(usr_da=turn['dialog_act'])
if i + 2 == len(sess):
dst.state['terminated'] = True
else:
for domain, svs in turn['sys_state'].items():
for slot, value in svs.items():
if slot != 'selectedResults':
dst.state['belief_state'][domain][
slot] = value
action = turn['dialog_act']
self.data[part].append([
self.vector.state_vectorize(deepcopy(dst.state)),
self.vector.action_vectorize(action)
])
dst.state['system_action'] = turn['dialog_act']
os.makedirs(processed_dir)
for part in ['train', 'val', 'test']:
with open(os.path.join(processed_dir, '{}.pkl'.format(part)),
'wb') as f:
pickle.dump(self.data[part], f)
def _load_data(self, processed_dir):
self.data = {}
for part in ['train', 'val', 'test']:
with open(os.path.join(processed_dir, '{}.pkl'.format(part)),
'rb') as f:
self.data[part] = pickle.load(f)
def create_dataset(self, part, batchsz):
print('Start creating {} dataset'.format(part))
s = []
a = []
for item in self.data[part]:
s.append(torch.Tensor(item[0]))
a.append(torch.Tensor(item[1]))
s = torch.stack(s)
a = torch.stack(a)
dataset = Dataset(s, a)
dataloader = data.DataLoader(dataset, batchsz, True)
print('Finish creating {} dataset'.format(part))
return dataloader