def save(self, save_dir): """ Saves :param save_dir: path to save to :type save_dir: str :return: into the void """ create_folder(save_dir) self.language_model.save(save_dir) for i, ph in enumerate(self.prediction_heads): ph.save(save_dir, i)
def save(self, save_dir): create_folder(save_dir) config = {} config["tokenizer"] = self.tokenizer.__class__.__name__ self.tokenizer.save_vocabulary(save_dir) # TODO make this generic to other tokenizers. We will probably want an own abstract Tokenizer config["lower_case"] = self.tokenizer.basic_tokenizer.do_lower_case config["max_seq_len"] = self.max_seq_len config["processor"] = self.__class__.__name__ output_config_file = os.path.join(save_dir, "processor_config.json") with open(output_config_file, "w") as file: json.dump(config, file)
def save(self, save_dir): """ Saves the language model and prediction heads. This will generate a config file and model weights for each. :param save_dir: path to save to :type save_dir: str """ create_folder(save_dir) self.language_model.save(save_dir) for i, ph in enumerate(self.prediction_heads): ph.save(save_dir, i)
def save(self, save_dir): """ Saves the vocabulary to file and also creates a json file containing all the information needed to load the same processor. :param save_dir: Directory where the files are to be saved :type save_dir: str """ create_folder(save_dir) config = {} config["tokenizer"] = self.tokenizer.__class__.__name__ self.tokenizer.save_vocabulary(save_dir) # TODO make this generic to other tokenizers. We will probably want an own abstract Tokenizer config["lower_case"] = self.tokenizer.basic_tokenizer.do_lower_case config["max_seq_len"] = self.max_seq_len config["processor"] = self.__class__.__name__ output_config_file = os.path.join(save_dir, "processor_config.json") with open(output_config_file, "w") as file: json.dump(config, file)