def run_and_get_hidden_activations(checkpoint_path, test_data_path, attention_method, use_attention_loss, ignore_output_eos, max_len=50, save_path=None): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=getattr(logging, 'INFO')) IGNORE_INDEX = -1 output_eos_used = not ignore_output_eos # load model logging.info("loading checkpoint from {}".format(os.path.join(checkpoint_path))) checkpoint = AnalysableSeq2seq.load(checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab # Prepare dataset and loss src = SourceField() tgt = TargetField(output_eos_used) tabular_data_fields = [('src', src), ('tgt', tgt)] if use_attention_loss or attention_method == 'hard': attn = AttentionField(use_vocab=False, ignore_index=IGNORE_INDEX) tabular_data_fields.append(('attn', attn)) src.vocab = input_vocab tgt.vocab = output_vocab tgt.eos_id = tgt.vocab.stoi[tgt.SYM_EOS] tgt.sos_id = tgt.vocab.stoi[tgt.SYM_SOS] def len_filter(example): return len(example.src) <= max_len and len(example.tgt) <= max_len # generate test set test = torchtext.data.TabularDataset( path=test_data_path, format='tsv', fields=tabular_data_fields, filter_pred=len_filter ) # When chosen to use attentive guidance, check whether the data is correct for the first # example in the data set. We can assume that the other examples are then also correct. if use_attention_loss or attention_method == 'hard': if len(test) > 0: if 'attn' not in vars(test[0]): raise Exception("AttentionField not found in test data") tgt_len = len(vars(test[0])['tgt']) - 1 # -1 for SOS attn_len = len(vars(test[0])['attn']) - 1 # -1 for preprended ignore_index if attn_len != tgt_len: raise Exception("Length of output sequence does not equal length of attention sequence in test data.") data_func = SupervisedTrainer.get_batch_data activations_dataset = run_model_on_test_data(model=seq2seq, data=test, get_batch_data=data_func) if save_path is not None: activations_dataset.save(save_path)
################################################################################# # load model logging.info("loading checkpoint from {}".format(os.path.join(opt.checkpoint_path))) checkpoint = Checkpoint.load(opt.checkpoint_path) seq2seq = checkpoint.model input_vocab = checkpoint.input_vocab output_vocab = checkpoint.output_vocab ############################################################################ # Prepare dataset and loss src = SourceField() tgt = TargetField() src.vocab = input_vocab tgt.vocab = output_vocab max_len = opt.max_len def len_filter(example): return len(example.src) <= max_len and len(example.tgt) <= max_len # generate test set test = torchtext.data.TabularDataset( path=opt.test_data, format='tsv', fields=[('src', src), ('tgt', tgt)], filter_pred=len_filter ) # Prepare loss weight = torch.ones(len(output_vocab)) pad = output_vocab.stoi[tgt.pad_token]
if opt.load_checkpoint is None: raise Exception( 'load_checkpoint must be specified when --resume is specified') else: logging.info("loading checkpoint from {}".format( os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint))) checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint) checkpoint = Checkpoint.load(checkpoint_path) seq2seq = checkpoint.model # input_vocab = checkpoint.input_vocab # output_vocab = checkpoint.output_vocab src.vocab = checkpoint.input_vocab tgt.vocab = checkpoint.output_vocab else: src.build_vocab(train, max_size=params['src_vocab_size'], specials=replace_tokens) tgt.build_vocab(train, max_size=params['tgt_vocab_size']) # input_vocab = src.vocab # output_vocab = tgt.vocab logging.info('Indices of special replace tokens:\n') for rep in replace_tokens: logging.info("%s, %d; " % (rep, src.vocab.stoi[rep])) logging.info('\n') # Prepare loss weight = torch.ones(len(tgt.vocab))
fields=[('src', src), ('tgt', tgt), ('beh', beh)], filter_pred=len_filter) dev = torchtext.data.TabularDataset(path=opt.dev_path, format='tsv', fields=[('src', src), ('tgt', tgt)], filter_pred=len_filter) if not os.path.exists(opt.ckpt_dir): os.makedirs(opt.ckpt_dir) if opt.resume: latest_checkpoint_path = Checkpoint.get_latest_checkpoint(opt.ckpt_dir) resume_checkpoint = Checkpoint.load(latest_checkpoint_path) src.vocab = resume_checkpoint.input_vocab tgt.vocab = resume_checkpoint.output_vocab else: print('Building vocab') #src.build_vocab(train, max_size=50000) #tgt.build_vocab(train, max_size=opt.vocab_size, vectors='glove.840B.300d') if hidden_size == 300: vectors = 'glove.42B.300d' elif hidden_size == 100: vectors = 'glove.6B.100d' else: vectors = None tgt.build_vocab(train, max_size=vocab_size, vectors=vectors) src.vocab = tgt.vocab input_vocab = src.vocab output_vocab = tgt.vocab