def __init__( self, vocab: Vocabulary, generator: Params, encoder: Params, samples: int, lambda_init: float, desired_length: float, rationale_extractor: Model = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): ''' The lagrangian relaxation code has been mostly copied from code accompanying original paper. ''' super(KumaraswamyGenEncClassifier, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._generator = Model.from_params( vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(generator), ) self._encoder = Model.from_params( vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(encoder), ) self._samples = samples self._desired_length = min(1.0, max(0.0, desired_length)) self._rationale_extractor = rationale_extractor self._loss_tracks = { k: Average() for k in [ "_lasso_loss", "_base_loss", "_rat_length", "_lambda0", "_c0_ma", "_c0" ] } s_min = torch.Tensor([-0.1]) s_max = torch.Tensor([1.1]) self.support = [s_min, s_max] self.lagrange_alpha = 0.9 self.lagrange_lr = 0.01 self.lambda_min = 1e-12 self.lambda_max = 5.0 self.register_buffer("lambda0", torch.full((1, ), lambda_init)) self.register_buffer("c0_ma", torch.full((1, ), 0.0)) # moving average initializer(self)
def __init__( self, vocab: Vocabulary, generator: Params, encoder: Params, samples: int, reg_loss_lambda: float, desired_length: float, reg_loss_mu: float, rationale_extractor: Model = None, supervise_rationale: bool = False, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(BernoulliGenEncClassifier, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._generator = Model.from_params( vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(generator), supervise_rationale=supervise_rationale, max_length_ratio=desired_length, ) self._encoder = Model.from_params( vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(encoder), ) self._samples = samples self._reg_loss_lambda = reg_loss_lambda self._reg_loss_mu = reg_loss_mu self._desired_length = min(1.0, max(0.0, desired_length)) self._rationale_extractor = rationale_extractor self._loss_tracks = { k: Average() for k in [ "_lasso_loss", "_base_loss", "_rat_length", "_fused_lasso_loss", "_censored_lasso_loss", "_generator_loss", ] } self._supervise_rationale = supervise_rationale initializer(self)
def from_params(cls, params: Params, vocab: Vocabulary, **extras) -> 'QaSrlParser': span_detector_params = params.pop("span_detector") question_predictor_params = params.pop("question_predictor") span_detector = Model.from_params(vocab=vocab, params=span_detector_params) question_predictor = Model.from_params( vocab=vocab, params=question_predictor_params) return QaSrlParser(vocab, span_detector=span_detector, question_predictor=question_predictor)
def __init__( self, vocab: Vocabulary, generator: Params, encoder: Params, samples: int, reg_loss_lambda: float, desired_length: float, reg_loss_mu: float = 2, rationale_extractor: Model = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(EncoderGeneratorModel, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._generator = Model.from_params(vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(generator)) self._encoder = Model.from_params(vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(encoder)) self._samples = samples self._reg_loss_lambda = reg_loss_lambda self._reg_loss_mu = reg_loss_mu self._desired_length = min(1.0, max(0.0, desired_length)) self._rationale_extractor = rationale_extractor self._loss_tracks = { k: Average() for k in [ "_lasso_loss", "_base_loss", "_rat_length", "_fused_lasso_loss", "_average_span_length" ] } s_min = torch.Tensor([-0.1]) s_max = torch.Tensor([1.1]) self.support = [s_min, s_max] # self.lagrange_alpha = 0.5 # self.lagrange_lr = 0.01 # self.register_buffer('lambda0', torch.full((1,), reg_loss_lambda)) # self.register_buffer('sparsity_ma', torch.full((1,), 0.)) # moving average initializer(self)
def __init__( self, vocab: Vocabulary, generator: Params, encoder: Params, samples: int, reg_loss_lambda: float, desired_length: float, reg_loss_mu: float, rationale_extractor: Model = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(KumaraswamyGenEncClassifier, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._generator = Model.from_params( vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(generator), ) self._encoder = Model.from_params( vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(encoder), ) self._samples = samples self._reg_loss_lambda = reg_loss_lambda self._reg_loss_mu = reg_loss_mu self._desired_length = min(1.0, max(0.0, desired_length)) self._rationale_extractor = rationale_extractor self._loss_tracks = { k: Average() for k in [ "_lasso_loss", "_base_loss", "_rat_length", "_fused_lasso_loss", "_censored_lasso_loss", "_generator_loss" ] } s_min = torch.Tensor([-0.1]) s_max = torch.Tensor([1.1]) self.support = [s_min, s_max] initializer(self)
def load_teacher_model(teacher_path=None, device=-1): models = {} if teacher_path is not None: for tea_path in teacher_path.split(","): # teacher path is something like "Models/HotpotQA,Models/SQuAD" tea_name = tea_path.split("/")[-1] config = Params.from_file(os.path.join(tea_path, CONFIG_NAME)) vocab_tea = Vocabulary.from_files( os.path.join(tea_path, "vocabulary")) model = Model.from_params(vocab=vocab_tea, params=config.get("model")) tea_model = copy.deepcopy(model) model_state = torch.load( os.path.join(tea_path, "best.th"), map_location=nn_util.device_mapping(cuda_device)) tea_model.load_state_dict(model_state) logger.info("Load teacher model from %s", tea_path) # freeze the parameters of teacher model for p in tea_model.parameters(): p.requires_grad = False if device >= 0: tea_model.to(device=device) models[tea_name] = tea_model return models
def _load(cls, config: Params, serialization_dir: str, weights_file: str = None, cuda_device: int = -1) -> 'Model': """ Ensembles don't have vocabularies or weights of their own, so they override _load. """ model_params = config.get('model') # The experiment config tells us how to _train_ a model, including where to get pre-trained # embeddings from. We're now _loading_ the model, so those embeddings will already be # stored in our weights. We don't need any pretrained weight file anymore, and we don't # want the code to look for it, so we remove it from the parameters here. remove_pretrained_embedding_params(model_params) model = Model.from_params(vocab=None, params=model_params) # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are # in sync with the weights if cuda_device >= 0: model.cuda(cuda_device) else: model.cpu() return model
def train(model_path, train_path, val_path, seed, vocabulary_path=None, config_path=None): assert os.path.isdir(model_path), "Model directory does not exist" set_seed(seed) config_path = config_path or os.path.join(model_path, "config.json") assert os.path.isfile(config_path), "Config file does not exist" params = Params.from_file(config_path) vocabulary_path = vocabulary_path or os.path.join(model_path, "vocabulary") assert os.path.exists(vocabulary_path), "Vocabulary is not ready, do not forget to run preprocess.py first" vocabulary = Vocabulary.from_files(vocabulary_path) reader_params = params.duplicate().pop("reader", default=Params({})) reader = DatasetReader.from_params(reader_params) train_dataset = reader.read(train_path) val_dataset = reader.read(val_path) if val_path else None model_params = params.pop("model") model = Model.from_params(model_params, vocab=vocabulary) print(model) print("Trainable params count: ", sum(p.numel() for p in model.parameters() if p.requires_grad)) iterator = DataIterator.from_params(params.pop('iterator')) iterator.index_with(vocabulary) trainer = Trainer.from_params(model, model_path, iterator, train_dataset, val_dataset, params.pop('trainer')) trainer.train()
def from_params(params: Params, iterator, val_iterator, vocab, all_datasets, serialization_dir: str, recover: bool = False) -> \ 'TrainerPieces': model = Model.from_params(vocab=vocab, params=params.pop('model')) # Initializing the model can have side effect of expanding the vocabulary vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") no_grad_regexes = trainer_params.pop("no_grad", ()) for name, parameter in model.named_parameters(): if any(re.search(regex, name) for regex in no_grad_regexes): parameter.requires_grad_(False) frozen_parameter_names, tunable_parameter_names = \ get_frozen_and_tunable_parameter_names(model) logger.info("Following parameters are Frozen (without gradient):") for name in frozen_parameter_names: logger.info(name) logger.info("Following parameters are Tunable (with gradient):") for name in tunable_parameter_names: logger.info(name) return TrainerPieces(model, iterator, train_data, validation_data, test_data, val_iterator, trainer_params)
def _load( cls, config: Params, serialization_dir: str, weights_file: Optional[str] = None, cuda_device: int = -1, opt_level: Optional[str] = None, ) -> Model: """ Ensembles don't have vocabularies or weights of their own, so they override _load. """ if opt_level is not None: raise NotImplementedError(f"{cls.__name__} does not support AMP yet.") model_params = config.get("model") # The experiment config tells us how to _train_ a model, including where to get pre-trained # embeddings from. We're now _loading_ the model, so those embeddings will already be # stored in our weights. We don't need any pretrained weight file anymore, and we don't # want the code to look for it, so we remove it from the parameters here. remove_weights_related_keys_from_params(model_params) model = Model.from_params(vocab=None, params=model_params) # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are # in sync with the weights if cuda_device >= 0: model.cuda(cuda_device) else: model.cpu() return model
def _load(cls, config: Params, serialization_dir: str, weights_file: str = None, cuda_device: int = -1) -> 'Model': """ Ensembles don't have vocabularies or weights of their own, so they override _load. """ model_params = config.get('model') # The experiment config tells us how to _train_ a model, including where to get pre-trained # embeddings from. We're now _loading_ the model, so those embeddings will already be # stored in our weights. We don't need any pretrained weight file anymore, and we don't # want the code to look for it, so we remove it from the parameters here. remove_pretrained_embedding_params(model_params) model = Model.from_params(vocab=None, params=model_params) # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are # in sync with the weights if cuda_device >= 0: model.cuda(cuda_device) else: model.cpu() return model
def from_params(params: Params, serialization_dir: str, recover: bool = False) -> 'TrainerPieces': all_datasets = training_util.datasets_from_params(params) datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets)) for dataset in datasets_for_vocab_creation: if dataset not in all_datasets: raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}") logger.info("From dataset instances, %s will be considered for vocabulary creation.", ", ".join(datasets_for_vocab_creation)) if recover and os.path.exists(os.path.join(serialization_dir, "vocabulary")): vocab = Vocabulary.from_files(os.path.join(serialization_dir, "vocabulary")) params.pop("vocabulary", {}) else: vocab = Vocabulary.from_params( params.pop("vocabulary", {}), (instance for key, dataset in all_datasets.items() for instance in dataset if key in datasets_for_vocab_creation) ) model = Model.from_params(vocab=vocab, params=params.pop('model')) # Initializing the model can have side effect of expanding the vocabulary vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(model.vocab) validation_iterator_params = params.pop("validation_iterator", None) if validation_iterator_params: validation_iterator = DataIterator.from_params(validation_iterator_params) validation_iterator.index_with(model.vocab) else: validation_iterator = None train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") no_grad_regexes = trainer_params.pop("no_grad", ()) for name, parameter in model.named_parameters(): if any(re.search(regex, name) for regex in no_grad_regexes): parameter.requires_grad_(False) frozen_parameter_names, tunable_parameter_names = \ get_frozen_and_tunable_parameter_names(model) logger.info("Following parameters are Frozen (without gradient):") for name in frozen_parameter_names: logger.info(name) logger.info("Following parameters are Tunable (with gradient):") for name in tunable_parameter_names: logger.info(name) return TrainerPieces(model, iterator, train_data, validation_data, test_data, validation_iterator, trainer_params)
def __init__( self, vocab: Vocabulary, generator: Params, encoder: Params, samples: int, reg_loss_lambda: float, desired_length: float, reg_loss_mu: float = 2, rationale_extractor: Model = None, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(EncoderGeneratorModel, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._generator = Model.from_params(vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(generator)) self._encoder = Model.from_params(vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(encoder)) self._samples = samples self._reg_loss_lambda = reg_loss_lambda self._reg_loss_mu = reg_loss_mu self._desired_length = min(1.0, max(0.0, desired_length)) self._rationale_extractor = rationale_extractor self._loss_tracks = { k: Average() for k in [ "_lasso_loss", "_base_loss", "_rat_length", "_fused_lasso_loss", "_average_span_length" ] } initializer(self)
def from_params(cls, vocab: Vocabulary, params: Params) -> 'GraphAndTextModel': text_model = params.pop("text_model", None) text_model = Model.from_params(vocab, text_model) graph_model = params.pop("graph_model") graph_model = Model.from_params(vocab, graph_model) classify_feed_forward = FeedForward.from_params( params.pop('classify_feed_forward')) initializer = InitializerApplicator.from_params( params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params( params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab=vocab, classify_feed_forward=classify_feed_forward, text_model=text_model, graph_model=graph_model, initializer=initializer, regularizer=regularizer)
def openie_model(serialization_dir, weights_file=None, cuda_device=-1): """ Instantiates an already-trained model, based on the experiment configuration and some optional overrides. """ # Load config config = Params.from_file(os.path.join(serialization_dir, CONFIG_NAME), "") config.loading_from_archive = True config = config.duplicate() weights_file = weights_file or os.path.join(serialization_dir, _WEIGHTS_NAME) # Load vocabulary from file vocab_dir = os.path.join(serialization_dir, "vocabulary") # If the config specifies a vocabulary subclass, we need to use it. vocab_params = config.get("vocabulary", Params({})) vocab_choice = vocab_params.pop_choice("type", Vocabulary.list_available(), True) vocab = Vocabulary.by_name(vocab_choice).from_files(vocab_dir) model_params = config.get("model") # The experiment config tells us how to _train_ a model, including where to get pre-trained # embeddings from. We're now _loading_ the model, so those embeddings will already be # stored in our weights. We don't need any pretrained weight file anymore, and we don't # want the code to look for it, so we remove it from the parameters here. remove_pretrained_embedding_params(model_params) model = Model.from_params(vocab=vocab, params=model_params) # If vocab+embedding extension was done, the model initialized from from_params # and one defined by state dict in weights_file might not have same embedding shapes. # Eg. when model embedder module was transferred along with vocab extension, the # initialized embedding weight shape would be smaller than one in the state_dict. # So calling model embedding extension is required before load_state_dict. # If vocab and model embeddings are in sync, following would be just a no-op. model.extend_embedder_vocab() model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device)) model.load_state_dict(model_state) # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are # in sync with the weights if cuda_device >= 0: model.cuda(cuda_device) else: model.cpu() return model, config
def __init__( self, vocab: Vocabulary, generator: Params, encoder: Params, reg_loss_lambda: float, reg_loss_mu: float = 2, reinforce_loss_weight: float = 1.0, rationale_supervision_loss_weight: float = 1.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None, ): super(EncoderGeneratorModel, self).__init__(vocab, initializer, regularizer) self._vocabulary = vocab self._num_labels = self._vocabulary.get_vocab_size("labels") self._generator = Model.from_params(vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(generator)) self._encoder = Model.from_params(vocab=vocab, regularizer=regularizer, initializer=initializer, params=Params(encoder)) self._reg_loss_lambda = reg_loss_lambda self._reg_loss_mu = reg_loss_mu self._reinforce_loss_weight = reinforce_loss_weight self._rationale_supervision_loss_weight = rationale_supervision_loss_weight self._loss_tracks = { k: Average() for k in ["lasso_loss", "base_loss", "fused_lasso_loss"] } initializer(self)
def from_params(cls, params: Params, vocab: Vocabulary) -> 'CMVMultiChannelPredictor': response_only_predictor = Model.from_params( params=params.pop('response_only_predictor'), vocab=vocab) op_response_predictor = Model.from_params( params=params.pop('op_response_predictor'), vocab=vocab) output_feedforward = FeedForward.from_params( params=params.pop('output_feedforward')) dropout = params.pop("dropout", 0) initializer = InitializerApplicator.from_params( params=params.pop('initializer', [])) regularizer = RegularizerApplicator.from_params( params=params.pop('regularizer', [])) params.assert_empty(cls.__name__) return cls(vocab, response_only_predictor, op_response_predictor, output_feedforward, dropout, initializer, regularizer)
def _get_predictor(args: argparse.Namespace) -> Predictor: check_for_gpu(args.cuda_device) params = Params.from_file(args.extractor_config_file) model = Model.from_params(vocab=None, params=params.pop('model')) if args.cuda_device >= 0: model.to(args.cuda_device) else: model.to(None) archive = Archive(model=model, config=params) return Predictor.from_archive( archive, args.predictor, dataset_reader_to_load=args.dataset_reader_choice)
def _test_model(self, file_name): params = self.params[file_name].duplicate() reader_params = params.duplicate().pop("reader", default=Params({})) if reader_params["type"] == "cnn_dailymail": reader_params["cnn_tokenized_dir"] = TEST_STORIES_DIR dataset_file = TEST_URLS_FILE elif reader_params["type"] == "ria": dataset_file = RIA_EXAMPLE_FILE else: assert False reader = DatasetReader.from_params(reader_params) tokenizer = reader._tokenizer dataset = reader.read(dataset_file) vocabulary_params = params.pop("vocabulary", default=Params({})) vocabulary = Vocabulary.from_params(vocabulary_params, instances=dataset) model_params = params.pop("model") model = Model.from_params(model_params, vocab=vocabulary) print(model) print("Trainable params count: ", sum(p.numel() for p in model.parameters() if p.requires_grad)) iterator = DataIterator.from_params(params.pop('iterator')) iterator.index_with(vocabulary) trainer = Trainer.from_params(model, None, iterator, dataset, None, params.pop('trainer')) trainer.train() model.eval() predictor = Seq2SeqPredictor(model, reader) for article, reference_sents in reader.parse_set(dataset_file): ref_words = [ token.text for token in tokenizer.tokenize(reference_sents) ] decoded_words = predictor.predict(article)["predicted_tokens"] self.assertGreaterEqual(len(decoded_words), len(ref_words)) unk_count = 0 while DEFAULT_OOV_TOKEN in decoded_words: unk_index = decoded_words.index(DEFAULT_OOV_TOKEN) decoded_words.pop(unk_index) unk_count += 1 if unk_index < len(ref_words): ref_words.pop(unk_index) self.assertLess(unk_count, 5) self.assertListEqual(decoded_words[:len(ref_words)], ref_words)
def _get_predictor(args: argparse.Namespace) -> Predictor: check_for_gpu(args.cuda_device) params = Params.from_file(args.scorer_config_file) archive = load_archive( args.archive_file, weights_file=args.weights_file, cuda_device=args.cuda_device, overrides=args.overrides, ) model = Model.from_params(vocab=None, model=archive.model, params=params) model.to(args.cuda_device) archive = Archive(model=model, config=archive.config) return Predictor.from_archive( archive, args.predictor, dataset_reader_to_load=args.dataset_reader_choice)
def instantiate_model_from_config(config_file_path: str, cuda_device: int = -1, overrides: str = None, include_package: str = "models") -> Model: logging.disable(logging.INFO) import_module_and_submodules(include_package) params = Params.from_file(config_file_path, overrides) vocab_dir = params.pop("vocabulary").pop("directory_path") vocab = Vocabulary.from_files(vocab_dir) model = Model.from_params(vocab=vocab, params=params.pop("model")) if cuda_device >= 0: model.cuda(cuda_device) else: model.cpu() return model
def load_model(model_name="conll_full_elmo"): """ Load both vocabulary and model and create and instance of HMTL full model. """ if model_name not in [ "conll_small_elmo", "conll_medium_elmo", "conll_full_elmo" ]: raise ValueError(f"{model_name} is not a valid name of model.") serialization_dir = "model_dumps" + "/" + model_name params = Params.from_file( params_file=os.path.join(serialization_dir, "config.json")) # Load TokenIndexer task_keys = [key for key in params.keys() if re.search("^task_", key)] token_indexer_params = params.pop(task_keys[-1]).pop("data_params").pop( "dataset_reader").pop("token_indexers") # see https://github.com/allenai/allennlp/issues/181 for better syntax token_indexers = {} for name, indexer_params in token_indexer_params.items(): token_indexers[name] = TokenIndexer.from_params(indexer_params) # Load the vocabulary logger.info("Loading Vocavulary from %s", os.path.join(serialization_dir, "vocabulary")) vocab = Vocabulary.from_files(os.path.join(serialization_dir, "vocabulary")) logger.info("Vocabulary loaded") # Create model and load weights model_params = params.pop("model") model = Model.from_params(vocab=vocab, params=model_params, regularizer=None) model_state_path = os.path.join(serialization_dir, "weights.th") model_state = torch.load(model_state_path, map_location="cpu") model.load_state_dict(state_dict=model_state) return model, vocab, token_indexers
def from_params( cls, # type: ignore params: Params, serialization_dir: str, recover: bool = False, ) -> "MultiTaskTrainer": readers = { name: DatasetReader.from_params(reader_params) for name, reader_params in params.pop( "train_dataset_readers").items() } train_file_paths = params.pop("train_file_paths").as_dict() datasets = { name: reader.read(train_file_paths[name]) for name, reader in readers.items() } instances = (instance for dataset in datasets.values() for instance in dataset) vocab = Vocabulary.from_params(Params({}), instances) model = Model.from_params(params.pop("model"), vocab=vocab) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(vocab) mingler = DatasetMingler.from_params(params.pop("mingler")) parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad] optimizer = Optimizer.from_params(parameters, params.pop("optimizer")) num_epochs = params.pop_int("num_epochs", 10) _ = params.pop("trainer", Params({})) params.assert_empty(__name__) return MultiTaskTrainer(model, serialization_dir, iterator, mingler, optimizer, datasets, num_epochs)
def modified_model_load(config: Params, serialization_dir: str, weights_file: str = None, cuda_device: int = -1) -> Model: """ Instantiates an already-trained model, based on the experiment configuration and some optional overrides. """ weights_file = weights_file or os.path.join(serialization_dir, _DEFAULT_WEIGHTS) # Load vocabulary from file vocab_dir = os.path.join(serialization_dir, 'vocabulary') # If the config specifies a vocabulary subclass, we need to use it. vocab = Vocabulary.from_files(vocab_dir) model_params = config.get('model') # The experiment config tells us how to _train_ a model, including where to get pre-trained # embeddings from. We're now _loading_ the model, so those embeddings will already be # stored in our weights. We don't need any pretrained weight file anymore, and we don't # want the code to look for it, so we remove it from the parameters here. remove_pretrained_embedding_params(model_params) model = Model.from_params(vocab=vocab, params=model_params) model_state = torch.load(weights_file, map_location=util.device_mapping(cuda_device)) model.load_state_dict(model_state, strict=False) # Force model to cpu or gpu, as appropriate, to make sure that the embeddings are # in sync with the weights if cuda_device >= 0: model.cuda(cuda_device) else: model.cpu() return model
def train_model(params: Params, serialization_dir: str, file_friendly_logging: bool = False, recover: bool = False) -> Model: """ Trains the model specified in the given :class:`Params` object, using the data and training parameters also specified in that object, and saves the results in ``serialization_dir``. Parameters ---------- params : ``Params`` A parameter object specifying an AllenNLP Experiment. serialization_dir : ``str`` The directory in which to save results and logs. file_friendly_logging : ``bool``, optional (default=False) If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow down tqdm's output to only once every 10 seconds. recover : ``bool`, optional (default=False) If ``True``, we will try to recover a training run from an existing serialization directory. This is only intended for use when something actually crashed during the middle of a run. For continuing training a model on new data, see the ``fine-tune`` command. """ prepare_environment(params) create_serialization_dir(params, serialization_dir, recover) prepare_global_logging(serialization_dir, file_friendly_logging) serialization_params = deepcopy(params).as_dict(quiet=True) with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file: json.dump(serialization_params, param_file, indent=4) all_datasets = datasets_from_params(params) datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets)) for dataset in datasets_for_vocab_creation: if dataset not in all_datasets: raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}") logger.info("Creating a vocabulary using %s data.", ", ".join(datasets_for_vocab_creation)) vocab = Vocabulary.from_params(params.pop("vocabulary", {}), (instance for key, dataset in all_datasets.items() for instance in dataset if key in datasets_for_vocab_creation)) vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) model = Model.from_params(vocab, params.pop('model')) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(vocab) train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") trainer = Trainer.from_params(model, serialization_dir, iterator, train_data, validation_data, trainer_params) evaluate_on_test = params.pop_bool("evaluate_on_test", False) params.assert_empty('base train command') try: metrics = trainer.train() except KeyboardInterrupt: # if we have completed an epoch, try to create a model archive. if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)): logging.info("Training interrupted by the user. Attempting to create " "a model archive using the current best epoch weights.") archive_model(serialization_dir, files_to_archive=params.files_to_archive) raise # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) if test_data and evaluate_on_test: test_metrics = evaluate(model, test_data, iterator, cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access for key, value in test_metrics.items(): metrics["test_" + key] = value elif test_data: logger.info("To evaluate on the test set after training, pass the " "'evaluate_on_test' flag, or use the 'allennlp evaluate' command.") metrics_json = json.dumps(metrics, indent=2) with open(os.path.join(serialization_dir, "metrics.json"), "w") as metrics_file: metrics_file.write(metrics_json) logger.info("Metrics: %s", metrics_json) return model
def train_model(params: Params, serialization_dir: str, file_friendly_logging: bool = False, recover: bool = False) -> Model: """ Trains the model specified in the given :class:`Params` object, using the data and training parameters also specified in that object, and saves the results in ``serialization_dir``. Parameters ---------- params : ``Params`` A parameter object specifying an AllenNLP Experiment. serialization_dir : ``str`` The directory in which to save results and logs. file_friendly_logging : ``bool``, optional (default=False) If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow down tqdm's output to only once every 10 seconds. recover : ``bool``, optional (default=False) If ``True``, we will try to recover a training run from an existing serialization directory. This is only intended for use when something actually crashed during the middle of a run. For continuing training a model on new data, see the ``fine-tune`` command. Returns ------- best_model: ``Model`` The model with the best epoch weights. """ prepare_environment(params) create_serialization_dir(params, serialization_dir, recover) prepare_global_logging(serialization_dir, file_friendly_logging) check_for_gpu(params.params.get('trainer').get('cuda_device', -1)) serialization_params = deepcopy(params).as_dict(quiet=True) with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file: json.dump(serialization_params, param_file, indent=4) all_datasets = datasets_from_params(params) datasets_for_vocab_creation = set( params.pop("datasets_for_vocab_creation", all_datasets)) for dataset in datasets_for_vocab_creation: if dataset not in all_datasets: raise ConfigurationError( f"invalid 'dataset_for_vocab_creation' {dataset}") logger.info("Creating a vocabulary using %s data.", ", ".join(datasets_for_vocab_creation)) vocab = Vocabulary.from_params( params.pop("vocabulary", {}), (instance for key, dataset in all_datasets.items() for instance in dataset if key in datasets_for_vocab_creation)) vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) model = Model.from_params(vocab=vocab, params=params.pop('model')) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(vocab) validation_iterator_params = params.pop("validation_iterator", None) if validation_iterator_params: validation_iterator = DataIterator.from_params( validation_iterator_params) validation_iterator.index_with(vocab) else: validation_iterator = None train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") no_grad_regexes = trainer_params.pop("no_grad", ()) for name, parameter in model.named_parameters(): if any(re.search(regex, name) for regex in no_grad_regexes): parameter.requires_grad_(False) frozen_parameter_names, tunable_parameter_names = \ get_frozen_and_tunable_parameter_names(model) logger.info("Following parameters are Frozen (without gradient):") for name in frozen_parameter_names: logger.info(name) logger.info("Following parameters are Tunable (with gradient):") for name in tunable_parameter_names: logger.info(name) trainer = Trainer.from_params(model, serialization_dir, iterator, train_data, validation_data, trainer_params, validation_iterator=validation_iterator) evaluate_on_test = params.pop_bool("evaluate_on_test", False) params.assert_empty('base train command') try: metrics = trainer.train() except KeyboardInterrupt: # if we have completed an epoch, try to create a model archive. if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)): logging.info( "Training interrupted by the user. Attempting to create " "a model archive using the current best epoch weights.") archive_model(serialization_dir, files_to_archive=params.files_to_archive) raise # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) logger.info("Loading the best epoch weights.") best_model_state_path = os.path.join(serialization_dir, 'best.th') best_model_state = torch.load(best_model_state_path) best_model = model best_model.load_state_dict(best_model_state) if test_data and evaluate_on_test: logger.info( "The model will be evaluated using the best epoch weights.") test_metrics = evaluate( best_model, test_data, validation_iterator or iterator, cuda_device=trainer._cuda_devices[0] # pylint: disable=protected-access ) for key, value in test_metrics.items(): metrics["test_" + key] = value elif test_data: logger.info( "To evaluate on the test set after training, pass the " "'evaluate_on_test' flag, or use the 'allennlp evaluate' command.") metrics_json = json.dumps(metrics, indent=2) with open(os.path.join(serialization_dir, "metrics.json"), "w") as metrics_file: metrics_file.write(metrics_json) logger.info("Metrics: %s", metrics_json) return best_model
def from_params(cls, params: Params, serialization_dir: str, recover: bool = False, cache_directory: str = None, cache_prefix: str = None) -> 'PtDistTrainer': all_datasets = training_util.datasets_from_params( params, cache_directory, cache_prefix) vocab = Vocabulary.from_files(params.vocabulary.directory_path) model = Model.from_params(vocab=vocab, params=params.pop('model')) model.extend_embedder_vocab() if is_master_rank(): vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) train_data = all_datasets['train'] validation_data = all_datasets.get('validation') batch_size = params.iterator.batch_size trainer_params = params.pop("trainer") keys = [key for key in params] for key in keys: params.pop(key) params = trainer_params validation_metric = params.pop("validation_metric", "-loss") shuffle = params.pop_bool("shuffle", True) num_epochs = params.pop_int("num_epochs", 20) cuda_device = parse_cuda_device(params.pop("cuda_device", -1)) grad_clipping = params.pop_float("grad_clipping", None) lr_scheduler_params = params.pop("learning_rate_scheduler", None) pretrain_file = params.pop("pretrain_file", None) no_grad_regexes = params.pop("no_grad", ()) for name, parameter in model.named_parameters(): if any(re.search(regex, name) for regex in no_grad_regexes): parameter.requires_grad_(False) frozen_parameter_names, tunable_parameter_names = \ get_frozen_and_tunable_parameter_names(model) logger.info("Following parameters are Frozen (without gradient):") for name in frozen_parameter_names: logger.info(name) logger.info("Following parameters are Tunable (with gradient):") for name in tunable_parameter_names: logger.info(name) model = model.cuda(dist.get_rank()) if pretrain_file: model_state = torch.load(pretrain_file, map_location=nn_util.device_mapping( dist.get_rank())) model.load_state_dict(model_state) parameters = [[n, p] for n, p in model.named_parameters() if p.requires_grad] # print([n for n, p in model.named_parameters() if p.requires_grad]) optimizer = Optimizer.from_params(parameters, params.pop("optimizer")) if lr_scheduler_params: lr_scheduler = LearningRateScheduler.from_params( optimizer, lr_scheduler_params) else: lr_scheduler = None num_serialized_models_to_keep = params.pop_int( "num_serialized_models_to_keep", 20) checkpointer = Checkpointer( serialization_dir=serialization_dir, num_serialized_models_to_keep=num_serialized_models_to_keep, keep_serialized_model_every_num_seconds=None) return cls(model, optimizer, train_data, validation_data, batch_size=batch_size, validation_metric=validation_metric, shuffle=shuffle, num_epochs=num_epochs, serialization_dir=serialization_dir, cuda_device=cuda_device, grad_clipping=grad_clipping, learning_rate_scheduler=lr_scheduler, checkpointer=checkpointer)
}}) params = Params.from_file(model_config, overrides) model_file = 'checkpoint/%s%s/' % (model_name, attn) iterator = DataIterator.from_params(params.pop("iterator")) torch.manual_seed(0) numpy.random.seed(0) if write_file: wf = Write_outfile(Wfile_name) print("Loading vocabulary") vocab = Vocabulary.from_files(model_file + 'vocabulary') print('Initialing model') model = Model.from_params(vocab=vocab, params=params.pop('model')) print("Loading Model file from %s" % (model_file + 'best.th')) with open(model_file + 'best.th', 'rb') as f: model.load_state_dict(torch.load(f, encoding='utf-8')) iterator.index_with(vocab) dataset_reader_params = params.pop('dataset_reader') datareader = DatasetReader.from_params(dataset_reader_params) model.eval() #读取文件数据 for file in files: dom = xml.dom.minidom.parse(file) root = dom.documentElement OrgQ_list = root.getElementsByTagName('OrgQuestion') q1_last = None
def train_model(params: Params, serialization_dir: str, file_friendly_logging: bool = False, recover: bool = False) -> Model: """ Trains the model specified in the given :class:`Params` object, using the data and training parameters also specified in that object, and saves the results in ``serialization_dir``. Parameters ---------- params : ``Params`` A parameter object specifying an AllenNLP Experiment. serialization_dir : ``str`` The directory in which to save results and logs. file_friendly_logging : ``bool``, optional (default=False) If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow down tqdm's output to only once every 10 seconds. recover : ``bool`, optional (default=False) If ``True``, we will try to recover a training run from an existing serialization directory. This is only intended for use when something actually crashed during the middle of a run. For continuing training a model on new data, see the ``fine-tune`` command. """ prepare_environment(params) create_serialization_dir(params, serialization_dir, recover) # TODO(mattg): pull this block out into a separate function (maybe just add this to # `prepare_environment`?) Tqdm.set_slower_interval(file_friendly_logging) sys.stdout = TeeLogger( os.path.join(serialization_dir, "stdout.log"), # type: ignore sys.stdout, file_friendly_logging) sys.stderr = TeeLogger( os.path.join(serialization_dir, "stderr.log"), # type: ignore sys.stderr, file_friendly_logging) handler = logging.FileHandler( os.path.join(serialization_dir, "python_logging.log")) handler.setLevel(logging.INFO) handler.setFormatter( logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s')) logging.getLogger().addHandler(handler) serialization_params = deepcopy(params).as_dict(quiet=True) with open(os.path.join(serialization_dir, CONFIG_NAME), "w") as param_file: json.dump(serialization_params, param_file, indent=4) all_datasets = datasets_from_params(params) datasets_for_vocab_creation = set( params.pop("datasets_for_vocab_creation", all_datasets)) for dataset in datasets_for_vocab_creation: if dataset not in all_datasets: raise ConfigurationError( f"invalid 'dataset_for_vocab_creation' {dataset}") logger.info("Creating a vocabulary using %s data.", ", ".join(datasets_for_vocab_creation)) vocab = Vocabulary.from_params( params.pop("vocabulary", {}), (instance for key, dataset in all_datasets.items() for instance in dataset if key in datasets_for_vocab_creation)) vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) model = Model.from_params(vocab, params.pop('model')) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(vocab) train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") trainer = Trainer.from_params(model, serialization_dir, iterator, train_data, validation_data, trainer_params) evaluate_on_test = params.pop_bool("evaluate_on_test", False) params.assert_empty('base train command') metrics = trainer.train() # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) if test_data and evaluate_on_test: test_metrics = evaluate(model, test_data, iterator, cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access for key, value in test_metrics.items(): metrics["test_" + key] = value elif test_data: logger.info( "To evaluate on the test set after training, pass the " "'evaluate_on_test' flag, or use the 'allennlp evaluate' command.") metrics_json = json.dumps(metrics, indent=2) with open(os.path.join(serialization_dir, "metrics.json"), "w") as metrics_file: metrics_file.write(metrics_json) logger.info("Metrics: %s", metrics_json) return model
def train_model(params: Params, serialization_dir: str, cuda_device: int, train_data_path: str, validation_data_path: str, test_data_path: str, file_friendly_logging: bool = False) -> Model: """ This function can be used as an entry point to running models in AllenNLP directly from a JSON specification using a :class:`Driver`. Note that if you care about reproducibility, you should avoid running code using Pytorch or numpy which affect the reproducibility of your experiment before you import and use this function, these libraries rely on random seeds which can be set in this function via a JSON specification file. Note that this function performs training and will also evaluate the trained model on development and test sets if provided in the parameter json. Parameters ---------- params: Params, required. A parameter object specifying an AllenNLP Experiment. serialization_dir: str, required The directory in which to save results and logs. """ prepare_environment(params) os.makedirs(serialization_dir, exist_ok=True) sys.stdout = TeeLogger( os.path.join(serialization_dir, "stdout.log"), # type: ignore sys.stdout, file_friendly_logging) sys.stderr = TeeLogger( os.path.join(serialization_dir, "stderr.log"), # type: ignore sys.stderr, file_friendly_logging) handler = logging.FileHandler( os.path.join(serialization_dir, "python_logging.log")) handler.setLevel(logging.INFO) handler.setFormatter( logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s')) logging.getLogger().addHandler(handler) serialization_params = deepcopy(params).as_dict(quiet=True) with open(os.path.join(serialization_dir, "model_params.json"), "w") as param_file: json.dump(serialization_params, param_file, indent=4) # all_datasets = datasets_from_params(params) all_datasets = datasets_from_args(params, train_data_path, validation_data_path, test_data_path) datasets_for_vocab_creation = set( params.pop("datasets_for_vocab_creation", all_datasets)) for dataset in datasets_for_vocab_creation: if dataset not in all_datasets: raise ConfigurationError( f"invalid 'dataset_for_vocab_creation' {dataset}") logger.info("Creating a vocabulary using %s data.", ", ".join(datasets_for_vocab_creation)) vocab = Vocabulary.from_params( params.pop("vocabulary", {}), (instance for key, dataset in all_datasets.items() for instance in dataset if key in datasets_for_vocab_creation)) vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) model = Model.from_params(vocab, params.pop('model')) if cuda_device >= 0: model = model.cuda(cuda_device) # iterator = DataIterator.from_params(params.pop("iterator")) # iterator.index_with(vocab) train_iterator = DataIterator.from_params(params.pop("train_iterator")) val_iterator = DataIterator.from_params(params.pop("val_iterator")) train_iterator.index_with(vocab) val_iterator.index_with(vocab) train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") trainer = Trainer.from_params(model, serialization_dir, train_iterator, val_iterator, cuda_device, train_data, validation_data, trainer_params) evaluate_on_test = params.pop_bool("evaluate_on_test", False) # params.assert_empty('base train command') metrics = trainer.train() # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) if test_data and evaluate_on_test: test_metrics = evaluate(model, test_data, val_iterator, cuda_device=trainer._cuda_devices[0]) # pylint: disable=protected-access for key, value in test_metrics.items(): metrics["test_" + key] = value elif test_data: logger.info( "To evaluate on the test set after training, pass the " "'evaluate_on_test' flag, or use the 'allennlp evaluate' command.") metrics_json = json.dumps(metrics, indent=2) with open(os.path.join(serialization_dir, "metrics.json"), "w") as metrics_file: metrics_file.write(metrics_json) logger.info("Metrics: %s", metrics_json) return model
def train_model(params: Params, serialization_dir: str, file_friendly_logging: bool = False, recover: bool = False, force: bool = False) -> Model: """ Trains the model specified in the given :class:`Params` object, using the data and training parameters also specified in that object, and saves the results in ``serialization_dir``. Parameters ---------- params : ``Params`` A parameter object specifying an AllenNLP Experiment. serialization_dir : ``str`` The directory in which to save results and logs. file_friendly_logging : ``bool``, optional (default=False) If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow down tqdm's output to only once every 10 seconds. recover : ``bool``, optional (default=False) If ``True``, we will try to recover a training run from an existing serialization directory. This is only intended for use when something actually crashed during the middle of a run. For continuing training a model on new data, see the ``fine-tune`` command. Returns ------- best_model: ``Model`` The model with the best epoch weights. """ prepare_environment(params) create_serialization_dir(params, serialization_dir, recover, force) prepare_global_logging(serialization_dir, file_friendly_logging) cuda_device = params.params.get('trainer').get('cuda_device', -1) if isinstance(cuda_device, list): for device in cuda_device: check_for_gpu(device) else: check_for_gpu(cuda_device) params.to_file(os.path.join(serialization_dir, CONFIG_NAME)) all_datasets = datasets_from_params(params) datasets_for_vocab_creation = set(params.pop("datasets_for_vocab_creation", all_datasets)) for dataset in datasets_for_vocab_creation: if dataset not in all_datasets: raise ConfigurationError(f"invalid 'dataset_for_vocab_creation' {dataset}") logger.info("From dataset instances, %s will be considered for vocabulary creation.", ", ".join(datasets_for_vocab_creation)) vocab = Vocabulary.from_params( params.pop("vocabulary", {}), (instance for key, dataset in all_datasets.items() for instance in dataset if key in datasets_for_vocab_creation) ) model = Model.from_params(vocab=vocab, params=params.pop('model')) # Initializing the model can have side effect of expanding the vocabulary vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) iterator = DataIterator.from_params(params.pop("iterator")) iterator.index_with(vocab) validation_iterator_params = params.pop("validation_iterator", None) if validation_iterator_params: validation_iterator = DataIterator.from_params(validation_iterator_params) validation_iterator.index_with(vocab) else: validation_iterator = None train_data = all_datasets['train'] validation_data = all_datasets.get('validation') test_data = all_datasets.get('test') trainer_params = params.pop("trainer") no_grad_regexes = trainer_params.pop("no_grad", ()) for name, parameter in model.named_parameters(): if any(re.search(regex, name) for regex in no_grad_regexes): parameter.requires_grad_(False) frozen_parameter_names, tunable_parameter_names = \ get_frozen_and_tunable_parameter_names(model) logger.info("Following parameters are Frozen (without gradient):") for name in frozen_parameter_names: logger.info(name) logger.info("Following parameters are Tunable (with gradient):") for name in tunable_parameter_names: logger.info(name) trainer_choice = trainer_params.pop_choice("type", Trainer.list_available(), default_to_first_choice=True) trainer = Trainer.by_name(trainer_choice).from_params(model=model, serialization_dir=serialization_dir, iterator=iterator, train_data=train_data, validation_data=validation_data, params=trainer_params, validation_iterator=validation_iterator) evaluate_on_test = params.pop_bool("evaluate_on_test", False) params.assert_empty('base train command') try: metrics = trainer.train() except KeyboardInterrupt: # if we have completed an epoch, try to create a model archive. if os.path.exists(os.path.join(serialization_dir, _DEFAULT_WEIGHTS)): logging.info("Training interrupted by the user. Attempting to create " "a model archive using the current best epoch weights.") archive_model(serialization_dir, files_to_archive=params.files_to_archive) raise # Now tar up results archive_model(serialization_dir, files_to_archive=params.files_to_archive) logger.info("Loading the best epoch weights.") best_model_state_path = os.path.join(serialization_dir, 'best.th') best_model_state = torch.load(best_model_state_path) best_model = model best_model.load_state_dict(best_model_state) if test_data and evaluate_on_test: logger.info("The model will be evaluated using the best epoch weights.") test_metrics = evaluate( best_model, test_data, validation_iterator or iterator, cuda_device=trainer._cuda_devices[0] # pylint: disable=protected-access ) for key, value in test_metrics.items(): metrics["test_" + key] = value elif test_data: logger.info("To evaluate on the test set after training, pass the " "'evaluate_on_test' flag, or use the 'allennlp evaluate' command.") dump_metrics(os.path.join(serialization_dir, "metrics.json"), metrics, log=True) return best_model
vocab = Vocabulary.from_files( os.path.join(args.serialization_dir, "vocabulary")) logger.info("Vocabulary loaded") ### Load the data iterators ### task_list = create_and_set_iterators(params=params, task_list=task_list, vocab=vocab) ### Regularization ### regularizer = None ### Create model ### model_params = params.pop("model") model = Model.from_params(vocab=vocab, params=model_params, regularizer=regularizer) ### Real evaluation ### cuda_device = params.pop("multi_task_trainer").pop_int("cuda_device", -1) metrics = {task._name: {} for task in task_list} for task in task_list: if not task._evaluate_on_test: continue logger.info("Task %s will be evaluated using the best epoch weights.", task._name) assert ( task._test_data is not None ), "Task {} wants to be evaluated on test dataset but no there is no test data loaded.".format(
def train_model(params: Params, serialization_dir: str) -> Model: """ This function can be used as an entry point to running models in AllenNLP directly from a JSON specification using a :class:`Driver`. Note that if you care about reproducibility, you should avoid running code using Pytorch or numpy which affect the reproducibility of your experiment before you import and use this function, these libraries rely on random seeds which can be set in this function via a JSON specification file. Note that this function performs training and will also evaluate the trained model on development and test sets if provided in the parameter json. Parameters ---------- params: Params, required. A parameter object specifying an AllenNLP Experiment. serialization_dir: str, required The directory in which to save results and logs. """ prepare_environment(params) os.makedirs(serialization_dir, exist_ok=True) sys.stdout = TeeLogger(os.path.join(serialization_dir, "stdout.log"), sys.stdout) # type: ignore sys.stderr = TeeLogger(os.path.join(serialization_dir, "stderr.log"), sys.stderr) # type: ignore handler = logging.FileHandler(os.path.join(serialization_dir, "python_logging.log")) handler.setLevel(logging.INFO) handler.setFormatter(logging.Formatter('%(asctime)s - %(levelname)s - %(name)s - %(message)s')) logging.getLogger().addHandler(handler) serialization_params = deepcopy(params).as_dict(quiet=True) with open(os.path.join(serialization_dir, "model_params.json"), "w") as param_file: json.dump(serialization_params, param_file, indent=4) # Now we begin assembling the required parts for the Trainer. dataset_reader = DatasetReader.from_params(params.pop('dataset_reader')) train_data_path = params.pop('train_data_path') logger.info("Reading training data from %s", train_data_path) train_data = dataset_reader.read(train_data_path) all_datasets: List[Dataset] = [train_data] datasets_in_vocab = ["train"] validation_data_path = params.pop('validation_data_path', None) if validation_data_path is not None: logger.info("Reading validation data from %s", validation_data_path) validation_data = dataset_reader.read(validation_data_path) all_datasets.append(validation_data) datasets_in_vocab.append("validation") else: validation_data = None test_data_path = params.pop("test_data_path", None) if test_data_path is not None: logger.info("Reading test data from %s", test_data_path) test_data = dataset_reader.read(test_data_path) all_datasets.append(test_data) datasets_in_vocab.append("test") else: test_data = None logger.info("Creating a vocabulary using %s data.", ", ".join(datasets_in_vocab)) vocab = Vocabulary.from_params(params.pop("vocabulary", {}), Dataset([instance for dataset in all_datasets for instance in dataset.instances])) vocab.save_to_files(os.path.join(serialization_dir, "vocabulary")) model = Model.from_params(vocab, params.pop('model')) iterator = DataIterator.from_params(params.pop("iterator")) train_data.index_instances(vocab) if validation_data: validation_data.index_instances(vocab) trainer_params = params.pop("trainer") trainer = Trainer.from_params(model, serialization_dir, iterator, train_data, validation_data, trainer_params) evaluate_on_test = params.pop("evaluate_on_test", False) params.assert_empty('base train command') trainer.train() # Now tar up results archive_model(serialization_dir) if test_data and evaluate_on_test: test_data.index_instances(vocab) evaluate(model, test_data, iterator, cuda_device=trainer._cuda_device) # pylint: disable=protected-access elif test_data: logger.info("To evaluate on the test set after training, pass the " "'evaluate_on_test' flag, or use the 'allennlp evaluate' command.") return model