action='store_true',
help=
'Check if the utterance is true given the KB. Only work for simulated data.'
)
add_scenario_arguments(parser)
add_lexicon_arguments(parser)
add_dataset_arguments(parser)
add_neural_system_arguments(parser)
add_heuristic_system_arguments(parser)
args = parser.parse_args()
logstats.init(args.stats_file)
if args.random_seed:
random.seed(args.random_seed)
np.random.seed(args.random_seed)
schema = Schema(args.schema_path)
scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path))
lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words)
if args.inverse_lexicon:
realizer = InverseLexicon(schema, args.inverse_lexicon)
else:
realizer = None
if args.train_max_examples is None:
args.train_max_examples = scenario_db.size
if args.test_max_examples is None:
args.test_max_examples = scenario_db.size
def get_system(name):
if name == 'simple':
for j in range(num_items):
if agent_items[0][i] == agent_items[1][j]:
matches.append((i, j))
if len(matches) == 0:
raise Exception('Internal error: expected at least one match, but got: %s' % matches)
# Create the scenario
kbs = [KB(scenario_attributes, items) for items in agent_items]
scenarios = []
for style in styles:
scenario = Scenario(generate_uuid('S'), scenario_attributes, kbs, style, [alphas[attr] for attr in scenario_attributes])
scenarios.append(scenario)
return scenarios
# Generate scenarios
schema = Schema(args.schema_path, args.domain)
scenario_list = []
while len(scenario_list) < args.num_scenarios * args.num_styles:
s_list = generate_scenario(schema)
for s in s_list:
if s is not None:
scenario_list.append(s)
scenario_db = ScenarioDB(scenario_list)
write_json(scenario_db.to_dict(), args.scenarios_path)
# Output a sample of what we've generated
for i in range(min(100, len(scenario_db.scenarios_list))):
print '---------------------------------------------------------------------------------------------'
print '---------------------------------------------------------------------------------------------'
scenario = scenario_db.scenarios_list[i]
else:
raise ValueError(
"Location of HTML templates should be specified in config with the key templates_dir"
)
if not os.path.exists(templates_dir):
raise ValueError("Specified HTML template location doesn't exist: %s" %
templates_dir)
app = create_app(debug=True, templates_dir=templates_dir)
schema_path = args.schema_path
if not os.path.exists(schema_path):
raise ValueError("No schema file found at %s" % schema_path)
schema = Schema(schema_path, domain=args.domain)
# todo in the future would we want individual models to have different lexicons?
lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words)
if args.inverse_lexicon:
realizer = InverseLexicon(schema, args.inverse_lexicon)
else:
realizer = None
scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path))
app.config['scenario_db'] = scenario_db
if 'models' not in params.keys():
params['models'] = {}
if 'quit_after' not in params.keys():
params[
'quit_after'] = params['status_params']['chat']['num_seconds'] + 1
parser.add_argument("--ranker-data", type=str, help="path to train data")
parser.add_argument("--annotated-examples-path",
help="Json of annotated examples",
type=str)
parser.add_argument("--scenarios-json",
help="Json of scenario information",
type=str)
parser.add_argument("--transcripts",
help="Json file of all transcripts collected")
parser.add_argument("--output", help="Output path")
add_lexicon_arguments(parser)
args = parser.parse_args()
path = args.schema
schema = Schema(path)
re_pattern = r"[\w*\']+|[(\w*&)]+|[\w]+|\.|\(|\)|\\|\"|\/|;|\#|\$|\%|\@|\{|\}|\:"
lexicon = Lexicon(schema,
learned_lex=False,
entity_ranker=None,
scenarios_json=args.scenarios_json,
stop_words=args.stop_words)
with open(args.annotated_examples_path, "r") as f:
annotated_examples = json.load(f)
with open(args.transcripts, "r") as f:
examples = json.load(f)
from src.basic.dataset import Example
parser = ArgumentParser()
add_scenario_arguments(parser)
add_lexicon_arguments(parser)
parser.add_argument('--transcripts',
type=str,
default='transcripts.json',
help='Path to directory containing transcripts')
parser.add_argument('--eval-transcripts',
type=str,
default='transcripts.json',
help='Path to directory containing transcripts')
parsed_args = parser.parse_args()
schema = Schema(parsed_args.schema_path)
scenario_db = ScenarioDB.from_dict(schema,
read_json(parsed_args.scenarios_path))
transcripts = read_json(parsed_args.transcripts)
eval_transcripts = read_json(parsed_args.eval_transcripts)
lexicon = Lexicon(schema,
False,
scenarios_json=parsed_args.scenarios_path,
stop_words=parsed_args.stop_words)
preprocessor = Preprocessor(schema, lexicon, 'canonical', 'canonical',
'canonical', False)
def compute_statistics(chats):
speech_act_summary_map = defaultdict(int)
total = 0.
assert ckpt.model_checkpoint_path, 'No model path found in checkpoint'
# Load vocab
mappings = read_pickle(vocab_path)
print 'Done [%fs]' % (time.time() - start)
else:
# Save config
if not os.path.isdir(args.checkpoint):
os.makedirs(args.checkpoint)
config_path = os.path.join(args.checkpoint, 'config.json')
write_json(vars(args), config_path)
model_args = args
mappings = None
ckpt = None
schema = Schema(model_args.schema_path, model_args.domain)
scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path))
dataset = read_dataset(scenario_db, args)
print 'Building lexicon...'
start = time.time()
lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words)
print '%.2f s' % (time.time() - start)
# Dataset
use_kb = False if model_args.model == 'encdec' else True
copy = True if model_args.model == 'attn-copy-encdec' else False
if model_args.model == 'attn-copy-encdec':
model_args.entity_target_form = 'graph'
preprocessor = Preprocessor(schema, lexicon,
model_args.entity_encoding_form,
model_args.entity_decoding_form,
templates_dir = None
if 'templates_dir' in params.keys():
templates_dir = params['templates_dir']
else:
raise ValueError("Location of HTML templates should be specified in config with the key templates_dir")
if not os.path.exists(templates_dir):
raise ValueError("Specified HTML template location doesn't exist: %s" % templates_dir)
app = create_app(debug=False, templates_dir=templates_dir)
schema_path = args.schema_path
if not os.path.exists(schema_path):
raise ValueError("No schema file found at %s" % schema_path)
schema = Schema(schema_path)
lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words)
if args.inverse_lexicon:
realizer = InverseLexicon(schema, args.inverse_lexicon)
else:
realizer = None
scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path))
app.config['scenario_db'] = scenario_db
if 'models' not in params.keys():
params['models'] = {}
if 'quit_after' not in params.keys():
params['quit_after'] = params['status_params']['chat']['num_seconds'] + 1
if 'skip_chat_enabled' not in params.keys():
