def extract_templates(self, transcripts_paths, max_examples=-1, ngram_N=4, log=None): examples = read_examples(transcripts_paths, max_examples, Scenario) for example in examples: if Preprocessor.skip_example(example): continue self.parse_example(example, ngram_N) self.add_counts(ngram_N) self.detokenize_templates() if log: self.log_examples_with_templates(examples, log)
parser.add_argument('--max-examples', default=-1, type=int) parser.add_argument('--templates', help='Path to load templates') parser.add_argument('--policy', help='Path to load model') parser.add_argument('--schema-path', help='Path to schema') parser.add_argument( '--agent', help='Only consider examples with the given type of agent') add_price_tracker_arguments(parser) args = parser.parse_args() lexicon = PriceTracker(args.price_tracker_model) #templates = Templates.from_pickle(args.templates) templates = Templates() manager = Manager.from_pickle(args.policy) analyzer = Analyzer(lexicon) # TODO: skip examples examples = read_examples(args.transcripts, args.max_examples, Scenario) agent = args.agent if agent is not None: examples = [e for e in examples if agent in e.agents.values()] analyzer.example_stats(examples, agent=agent) #import sys; sys.exit() parsed_dialogues = [] for example in examples: utterances = parse_example(example, lexicon, templates) parsed_dialogues.append(utterances) analyzer.parser_stats(parsed_dialogues, agent=agent) #analyzer.manager_stats(manager)
parser.add_argument('--transcripts', nargs='+', default=[]) parser.add_argument( '--max-examples', type=int, help='Maximum number of examples to read from transcripts') parser.add_argument('--output', help='Path to output model') parser.add_argument('--test', action='store_true') add_price_tracker_arguments(parser) add_slot_detector_arguments(parser) args = parser.parse_args() lexicon = PriceTracker(args.price_tracker_model) schema = Schema(args.schema_path) preprocessor = Preprocessor(schema, lexicon, 'canonical', 'canonical', 'canonical') examples = read_examples(None, args.transcripts, args.max_examples) if args.test: slot_detector = SlotDetector(slot_scores_path=args.slot_scores) print sorted( slot_detector.slot_scores['seller']['furniture']['hi'].items(), key=lambda x: x[1], reverse=True)[:10] #import sys; sys.exit() for ex in examples: kbs = ex.scenario.kbs kbs[1].dump() for e in ex.events: kb = kbs[e.agent] utterance = preprocessor.process_event(e, e.agent, kb) print ' '.join([
def examples(schema): data_paths = ['data/negotiation/dev.json'] return read_examples(None, data_paths, 10)