if "eval" in args.mode: if not os.path.exists(args.eval_file): raise FileNotFoundError( "eval data not found. Datasets can be obtained using examples/nlp/question_answering/get_squad.py" ) if "test" in args.mode: if not os.path.exists(args.test_file): raise FileNotFoundError( "test data not found. Datasets can be obtained using examples/nlp/question_answering/get_squad.py" ) # Instantiate neural factory with supported backend nf = nemo_core.NeuralModuleFactory( local_rank=args.local_rank, optimization_level=args.amp_opt_level, log_dir=args.work_dir, create_tb_writer=True, files_to_copy=[__file__], add_time_to_log_dir=False, ) model = nemo_nlp.nm.trainables.get_pretrained_lm_model( pretrained_model_name=args.pretrained_model_name, config=args.bert_config, vocab=args.vocab_file, checkpoint=args.bert_checkpoint, ) tokenizer = nemo.collections.nlp.data.tokenizers.get_tokenizer( tokenizer_name=args.tokenizer, pretrained_model_name=args.pretrained_model_name, tokenizer_model=args.tokenizer_model,
if not exists(abs_data_dir): raise ValueError(f"Folder `{abs_data_dir}` not found") # Prepare the experiment (output) dir. abs_work_dir = f'{expanduser(args.work_dir)}/dst_trade/' logging.info( "Logging the results of the experiment to: `{}`".format(abs_work_dir)) print("abs_data_dir = ", abs_data_dir) data_desc = MultiWOZDataDesc(abs_data_dir, domains) nf = nemo_core.NeuralModuleFactory( backend=nemo_core.Backend.PyTorch, local_rank=args.local_rank, optimization_level=args.amp_opt_level, log_dir=abs_work_dir, create_tb_writer=True, files_to_copy=[__file__], add_time_to_log_dir=True, ) vocab_size = len(data_desc.vocab) encoder = EncoderRNN(vocab_size, args.emb_dim, args.hid_dim, args.dropout, args.n_layers) decoder = TRADEGenerator( data_desc.vocab, encoder.embedding, args.hid_dim, args.dropout, data_desc.slots,