def dump_data(opt):
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
ignorefields = opt['ignore_fields'].split(',')
print("[ starting to convert.. ]")
fw = open(opt['outfile'], 'w')
for _ in range(opt['num_examples']):
world.parley()
world.acts[0]['labels'] = world.acts[0].get(
'labels', world.acts[0].pop('eval_labels', None))
txt = msg_to_str(world.acts[0], ignore_fields=ignorefields)
fw.write(txt + "\n")
if world.acts[0].get('episode_done', False):
fw.write("\n")
if world.epoch_done():
print('EPOCH DONE')
break
fw.close()
def dump_data(opt):
# create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
ignorefields = opt.get('ignore_fields', '')
if opt['outfile'] is None:
outfile = tempfile.mkstemp(prefix='{}_{}_'.format(
opt['task'], opt['datatype']),
suffix='.txt')[1]
else:
outfile = opt['outfile']
if opt['num_examples'] == -1:
num_examples = world.num_examples()
else:
num_examples = opt['num_examples']
log_timer = TimeLogger()
print('[ starting to convert.. ]')
print('[ saving output to {} ]'.format(outfile))
fw = open(outfile, 'w')
for _ in range(num_examples):
world.parley()
world.acts[0]['labels'] = world.acts[0].get(
'labels', world.acts[0].pop('eval_labels', None))
txt = msg_to_str(world.acts[0], ignore_fields=ignorefields)
fw.write(txt + '\n')
if world.acts[0].get('episode_done', False):
fw.write('\n')
if log_timer.time() > opt['log_every_n_secs']:
text, _log = log_timer.log(world.total_parleys,
world.num_examples())
print(text)
if world.epoch_done():
print('EPOCH DONE')
break
fw.close()
def make_dataset(opt):
# Initialize control information so we can compute sentence attributes.
# Here we set build_task=False so we don't download data/controllable_dialogue
# (because we're trying to create it instead).
initialize_control_information(opt, build_task=False)
# Create repeat label agent and assign it to the specified task
agent = RepeatLabelAgent(opt)
world = create_task(opt, agent)
ignorefields = opt.get('ignore_fields', '')
outfile = opt['outfile']
# Number of examples to process
if opt['num_examples'] == -1:
num_examples = world.num_examples()
else:
num_examples = opt['num_examples']
# List of controls to include:
controls = opt['controls'].split(',') if opt['controls'] != '' else []
print('[ starting to convert.. ]')
print('[ saving output to {} ]'.format(outfile))
fw = open(outfile, 'w')
log_timer = TimeLogger()
for _ in range(num_examples):
world.parley()
world.acts[0]['labels'] = world.acts[0].get(
'labels', world.acts[0].pop('eval_labels', None))
# Need to get history in order to compute control values
hist = ConvAI2History(world.acts[0]['text'], assume_persontokens=False)
response = world.acts[0]['labels'][0]
# Compute control values
for ctrl in controls:
ctrl_val = eval_attr(response, hist, ctrl)
if ctrl == 'avg_nidf':
assert ctrl_val >= 0
assert ctrl_val <= 1
elif ctrl == 'question':
assert ctrl_val in [0, 1]
elif ctrl == 'lastuttsim':
if ctrl_val is not None:
assert ctrl_val >= -1
assert ctrl_val <= 1
else:
raise Exception('unexpected ctrl name: %s' % ctrl)
world.acts[0][ctrl] = ctrl_val # add control value to act
# Write to file
txt = msg_to_str(world.acts[0], ignore_fields=ignorefields)
fw.write(txt + '\n')
if world.acts[0].get('episode_done', False):
fw.write('\n')
if log_timer.time() > opt['log_every_n_secs']:
text, _log = log_timer.log(world.total_parleys,
world.num_examples())
print(text)
if world.epoch_done():
print('EPOCH DONE')
break
fw.close()
def write_dialog(opt, fw, d, label_type, split):
l = len(d['speech'])
msgs = []
text = '_task_' + label_type + '\n'
if opt['light_use_setting']:
text += ('_setting_name ' + d['setting']['name'] + ", " +
d['setting']['category'] + '\n')
text += '_setting_desc ' + d['setting']['description'] + '\n'
if opt['light_use_person_names']:
text += '_partner_name ' + d['partner_agent']['name'] + '\n'
text += '_self_name ' + d['self_agent']['name'] + '\n'
if use_feat(opt, 'light_use_persona', 'self'):
text += '_self_persona ' + d['self_agent']['persona'] + '\n'
if use_feat(opt, 'light_use_persona', 'other'):
text += '_other_persona ' + d['partner_agent']['persona'] + '\n'
if opt['light_use_objects']:
for o, od in d['all_descriptions'].items():
text += '_object_desc ' + o + " : " + od + '\n'
if False:
for o in d['room_objects'][0]:
text += '_object_in_room ' + o + '\n'
for o in d['carrying'][0]:
text += '_object_carrying ' + o + '\n'
for o in d['wearing'][0]:
text += '_object_wearing ' + o + '\n'
for o in d['wielding'][0]:
text += '_object_wielding ' + o + '\n'
for i in range(0, l, 2):
if i < l - 1:
if (use_feat(opt, 'light_use_speech', 'partner')
and d['speech'][i] is not None):
text += '_partner_say ' + str(d['speech'][i]) + '\n'
if (use_feat(opt, 'light_use_action', 'partner')
and d['action'][i] is not None):
text += '_partner_act ' + str(d['action'][i]) + '\n'
if (use_feat(opt, 'light_use_emote', 'partner')
and d['emote'][i] is not None):
text += '_partner_emote ' + str(d['emote'][i]) + '\n'
if opt.get('light_use_repeat') == 'self_last':
if i > 0:
text = str(d['speech'][i - 1])
else:
text = 'nothing'
if opt.get('light_use_repeat') == 'partner_last':
text = str(d['speech'][i])
if opt.get('light_use_repeat') == 'both_last':
text = ''
if i > 0:
text += str(d['speech'][i - 1]) + ' '
text += str(d['speech'][i])
label = d[label_type][i + 1]
used_current = False
shown = {}
if (use_feat(opt, 'light_use_current_self_output', 'all')
and label_type != 'speech'
and use_feat(opt, 'light_use_speech', 'self')
and d['speech'][i + 1] is not None):
if 'remove' not in opt['light_use_current_self_output']:
text += '_self_say ' + str(d['speech'][i + 1]) + '\n'
shown['speech'] = True
used_current = True
if (use_feat(opt, 'light_use_current_self_output', 'all')
and label_type != 'action'
and use_feat(opt, 'light_use_action', 'self')
and d['action'][i + 1] is not None):
if 'remove' not in opt['light_use_current_self_output']:
text += '_self_act ' + str(d['action'][i + 1]) + '\n'
shown['action'] = True
used_current = True
if (use_feat(opt, 'light_use_current_self_output', 'all')
and label_type != 'emote'
and use_feat(opt, 'light_use_emote', 'self')
and d['emote'][i + 1] is not None):
if 'remove' not in opt['light_use_current_self_output']:
text += '_self_emote ' + str(d['emote'][i + 1]) + '\n'
shown['emote'] = True
used_current = True
if ('all_filtered' in opt['light_use_current_self_output']
and used_current is False):
label = None
if label is not None:
msg = {}
msg['text'] = text
msg['labels'] = label
add_negs(
msg,
d,
i + 1,
label_type,
split,
int(opt.get('light_use_cands', 100)),
opt.get('light_use_affordances', True),
)
msgs.append(msg)
text = ''
if (use_feat(opt, 'light_use_speech', 'self')
and d['speech'][i + 1] is not None
and ('speech' not in shown)):
text += '_self_say ' + str(d['speech'][i + 1]) + '\n'
if (use_feat(opt, 'light_use_action', 'self')
and d['action'][i + 1] is not None
and ('action' not in shown)):
text += '_self_act ' + str(d['action'][i + 1]) + '\n'
if (use_feat(opt, 'light_use_emote', 'self')
and d['emote'][i + 1] is not None
and ('emote' not in shown)):
text += '_self_emote ' + str(d['emote'][i + 1]) + '\n'
if len(msgs) > 0:
msgs[-1]['episode_done'] = True
for m in msgs:
# print(m.replace('\n', '\\n'))
fix_labels(m, opt)
global mx
mx = max(len(m['label_candidates']), mx)
# print(mx)
txt = msg_to_str(m)
fw.write(txt + '\n')