action='store_true', help= 'Check if the utterance is true given the KB. Only work for simulated data.' ) add_scenario_arguments(parser) add_lexicon_arguments(parser) add_dataset_arguments(parser) add_neural_system_arguments(parser) add_heuristic_system_arguments(parser) args = parser.parse_args() logstats.init(args.stats_file) if args.random_seed: random.seed(args.random_seed) np.random.seed(args.random_seed) schema = Schema(args.schema_path) scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path)) lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words) if args.inverse_lexicon: realizer = InverseLexicon(schema, args.inverse_lexicon) else: realizer = None if args.train_max_examples is None: args.train_max_examples = scenario_db.size if args.test_max_examples is None: args.test_max_examples = scenario_db.size def get_system(name): if name == 'simple':
for j in range(num_items): if agent_items[0][i] == agent_items[1][j]: matches.append((i, j)) if len(matches) == 0: raise Exception('Internal error: expected at least one match, but got: %s' % matches) # Create the scenario kbs = [KB(scenario_attributes, items) for items in agent_items] scenarios = [] for style in styles: scenario = Scenario(generate_uuid('S'), scenario_attributes, kbs, style, [alphas[attr] for attr in scenario_attributes]) scenarios.append(scenario) return scenarios # Generate scenarios schema = Schema(args.schema_path, args.domain) scenario_list = [] while len(scenario_list) < args.num_scenarios * args.num_styles: s_list = generate_scenario(schema) for s in s_list: if s is not None: scenario_list.append(s) scenario_db = ScenarioDB(scenario_list) write_json(scenario_db.to_dict(), args.scenarios_path) # Output a sample of what we've generated for i in range(min(100, len(scenario_db.scenarios_list))): print '---------------------------------------------------------------------------------------------' print '---------------------------------------------------------------------------------------------' scenario = scenario_db.scenarios_list[i]
else: raise ValueError( "Location of HTML templates should be specified in config with the key templates_dir" ) if not os.path.exists(templates_dir): raise ValueError("Specified HTML template location doesn't exist: %s" % templates_dir) app = create_app(debug=True, templates_dir=templates_dir) schema_path = args.schema_path if not os.path.exists(schema_path): raise ValueError("No schema file found at %s" % schema_path) schema = Schema(schema_path, domain=args.domain) # todo in the future would we want individual models to have different lexicons? lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words) if args.inverse_lexicon: realizer = InverseLexicon(schema, args.inverse_lexicon) else: realizer = None scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path)) app.config['scenario_db'] = scenario_db if 'models' not in params.keys(): params['models'] = {} if 'quit_after' not in params.keys(): params[ 'quit_after'] = params['status_params']['chat']['num_seconds'] + 1
parser.add_argument("--ranker-data", type=str, help="path to train data") parser.add_argument("--annotated-examples-path", help="Json of annotated examples", type=str) parser.add_argument("--scenarios-json", help="Json of scenario information", type=str) parser.add_argument("--transcripts", help="Json file of all transcripts collected") parser.add_argument("--output", help="Output path") add_lexicon_arguments(parser) args = parser.parse_args() path = args.schema schema = Schema(path) re_pattern = r"[\w*\']+|[(\w*&)]+|[\w]+|\.|\(|\)|\\|\"|\/|;|\#|\$|\%|\@|\{|\}|\:" lexicon = Lexicon(schema, learned_lex=False, entity_ranker=None, scenarios_json=args.scenarios_json, stop_words=args.stop_words) with open(args.annotated_examples_path, "r") as f: annotated_examples = json.load(f) with open(args.transcripts, "r") as f: examples = json.load(f)
from src.basic.dataset import Example parser = ArgumentParser() add_scenario_arguments(parser) add_lexicon_arguments(parser) parser.add_argument('--transcripts', type=str, default='transcripts.json', help='Path to directory containing transcripts') parser.add_argument('--eval-transcripts', type=str, default='transcripts.json', help='Path to directory containing transcripts') parsed_args = parser.parse_args() schema = Schema(parsed_args.schema_path) scenario_db = ScenarioDB.from_dict(schema, read_json(parsed_args.scenarios_path)) transcripts = read_json(parsed_args.transcripts) eval_transcripts = read_json(parsed_args.eval_transcripts) lexicon = Lexicon(schema, False, scenarios_json=parsed_args.scenarios_path, stop_words=parsed_args.stop_words) preprocessor = Preprocessor(schema, lexicon, 'canonical', 'canonical', 'canonical', False) def compute_statistics(chats): speech_act_summary_map = defaultdict(int) total = 0.
assert ckpt.model_checkpoint_path, 'No model path found in checkpoint' # Load vocab mappings = read_pickle(vocab_path) print 'Done [%fs]' % (time.time() - start) else: # Save config if not os.path.isdir(args.checkpoint): os.makedirs(args.checkpoint) config_path = os.path.join(args.checkpoint, 'config.json') write_json(vars(args), config_path) model_args = args mappings = None ckpt = None schema = Schema(model_args.schema_path, model_args.domain) scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path)) dataset = read_dataset(scenario_db, args) print 'Building lexicon...' start = time.time() lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words) print '%.2f s' % (time.time() - start) # Dataset use_kb = False if model_args.model == 'encdec' else True copy = True if model_args.model == 'attn-copy-encdec' else False if model_args.model == 'attn-copy-encdec': model_args.entity_target_form = 'graph' preprocessor = Preprocessor(schema, lexicon, model_args.entity_encoding_form, model_args.entity_decoding_form,
templates_dir = None if 'templates_dir' in params.keys(): templates_dir = params['templates_dir'] else: raise ValueError("Location of HTML templates should be specified in config with the key templates_dir") if not os.path.exists(templates_dir): raise ValueError("Specified HTML template location doesn't exist: %s" % templates_dir) app = create_app(debug=False, templates_dir=templates_dir) schema_path = args.schema_path if not os.path.exists(schema_path): raise ValueError("No schema file found at %s" % schema_path) schema = Schema(schema_path) lexicon = Lexicon(schema, args.learned_lex, stop_words=args.stop_words) if args.inverse_lexicon: realizer = InverseLexicon(schema, args.inverse_lexicon) else: realizer = None scenario_db = ScenarioDB.from_dict(schema, read_json(args.scenarios_path)) app.config['scenario_db'] = scenario_db if 'models' not in params.keys(): params['models'] = {} if 'quit_after' not in params.keys(): params['quit_after'] = params['status_params']['chat']['num_seconds'] + 1 if 'skip_chat_enabled' not in params.keys():